You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
yanzong/app/job/service/goods/Collector.php

643 lines
27 KiB

<?php
// +----------------------------------------------------------------------
// | 萤火商城系统 [ 致力于通过产品和服务,帮助商家高效化开拓市场 ]
// +----------------------------------------------------------------------
// | Copyright (c) 2017~2023 https://www.yiovo.com All rights reserved.
// +----------------------------------------------------------------------
// | Licensed 这不是一个自由软件,不允许对程序代码以任何形式任何目的的再发行
// +----------------------------------------------------------------------
// | Author: 萤火科技 <admin@yiovo.com>
// +----------------------------------------------------------------------
declare (strict_types=1);
namespace app\job\service\goods;
use app\store\service\Upload as UploadService;
use app\common\service\BaseService;
use app\common\model\Spec as SpecModel;
use app\common\model\Goods as GoodsModel;
use app\common\model\GoodsSku as GoodsSkuModel;
use app\common\model\GoodsImage as GoodsImageModel;
use app\store\model\GoodsImage as GoodsImageModel1;
use app\common\model\store\Setting as SettingModel;
use app\common\model\GoodsSpecRel as GoodsSpecRelModel;
use app\common\model\goods\Collector as GoodsCollectorModel;
use app\common\model\GoodsCategoryRel as GoodsCategoryRelModel;
use app\common\enum\Setting as SettingEnum;
use app\common\enum\file\FileType as FileTypeEnum;
use app\common\enum\goods\SpecType as GoodsSpecTypeEnum;
use app\common\enum\goods\DeductStockType as DeductStockTypeEnum;
use app\common\enum\goods\CollectorStatus as GoodsCollectorStatusEnum;
use app\common\library\helper;
use app\common\library\Download;
use app\common\library\collector\tools\UrlStore;
use app\common\library\collector\tools\UrlItemId;
use app\common\library\collector\Facade as CollectorFacade;
use cores\exception\BaseException;
/**
* 服务类:商品批量采集
* Class Collector
* @package app\job\service\goods
*/
class Collector extends BaseService
{
/**
* 商品采集记录
* @var GoodsCollectorModel|null
*/
private ?GoodsCollectorModel $record = null;
/**
* 错误日志记录
* @var array
*/
private array $errorLog = [];
/**
* 采集成功的数量
* @var int
*/
private int $successCount = 0;
/**
* 图片url临时记录 (同一个网络图片仅入库一次)
* @var array
*/
private array $dictImageUrls = [];
/**
* 批量采集商品
* @param array $urls 采集的商品url集
* @param array $form 用户提交的表单
* @param int $recordId 采集记录id
* @param int $storeId 当前商城id
* @return bool
* @throws BaseException
* @throws \think\db\exception\DataNotFoundException
* @throws \think\db\exception\DbException
* @throws \think\db\exception\ModelNotFoundException
*/
public function batch(array $urls, array $form, int $recordId, int $storeId): bool
{
// 检查采集记录状态是否异常
if (!$this->checkStatusFail($recordId)) {
foreach ($urls as $url) {
try {
// 采集第三方商品数据
$original = $this->collector($url, $storeId);
if ($original['spec_type'] == 20) {
$original['spec_type'] = 10;
$skuList = array_column($original['specData']['skuList'], null, "goods_sku_no");
$goods_price = $skuList[$original['goods_sku_no']]['goods_price'] ?? 0;
$original['goods_price'] = $goods_price;
$original['line_price'] = $goods_price;
$original['data_type'] = 1;
$original['link'] = $url;
unset($original['specData']);
}
// 下载远程商品图片
$original = $this->thirdPartyImages($original, $form['imageStorage'], $storeId);
} catch (\Throwable $e) {
tre($e->getTraceAsString());
$this->errorLog[] = ['url' => trim($url), 'message' => $e->getMessage()];
continue;
}
// 生成商品数据(用于写入数据库)
$data = $this->createData($original, $form, $storeId);
// 事务处理:添加商品
$model = new GoodsModel();
$model->transaction(function () use ($model, $data, $storeId) {
// 添加商品
$model->save($data);
$model->where('goods_id', (int)$model['goods_id'])->update(['spu_id' => (int)$model['goods_id']]);
// 新增商品与分类关联
GoodsCategoryRelModel::increased((int)$model['goods_id'], $data['categoryIds'], $storeId);
// 新增商品与图片关联
GoodsImageModel::increased((int)$model['goods_id'], $data['imagesIds'], $storeId);
// 新增商品与规格关联
GoodsSpecRelModel::increased((int)$model['goods_id'], $data['newSpecList'], $storeId);
// 新增商品sku信息
GoodsSkuModel::add((int)$model['goods_id'], $data['spec_type'], $data['newSkuList'], $storeId);
});
// 记录采集成功
$this->successCount++;
}
}
// 更新采集记录
$this->updateRecord($recordId, \count($urls));
return true;
}
/**
* 后台单个采集
* [single description]
* @param string $url [description]
* @param array $form [description]
* @param int $storeId [description]
* @return [type] [description]
*/
public function single(string $url, array $form, int $storeId): bool
{
try {
// 采集第三方商品数据
$original = $this->collector($url, $storeId);
if ($original['spec_type'] == 20) {
$original['spec_type'] = 10;
unset($original['specData']);
}
$original['goods_price'] = $form['goods_price'] ?? 0.00;
$original['line_price'] = $form['goods_price'] ?? 0.00;
$original['cost_price'] = $form['cost_price'] ?? 0.00;
$original['stock_num'] = $form['stock_num'] ?? 0;
$original['remark'] = $form['remark'] ?? "";
$original['cmmdty_model'] = $form['cmmdty_model'] ?? "";
$original['goods_no'] = $form['goods_no'] ?? "";
$original['region'] = $form['region'] ?? "";
$original['region_text'] = $form['region_text'] ?? "";
$original['data_type'] = 1;
$original['link'] = $url;
// 下载远程商品图片
$original = $this->thirdPartyImages($original, $form['imageStorage'], $storeId);
} catch (\Throwable $e) {
// var_dump($e->getMessage());
// exit;
tre($e->getTraceAsString());
$this->errorLog[] = ['url' => trim($url), 'message' => $e->getMessage()];
return false;
}
// 生成商品数据(用于写入数据库)
$data = $this->singleCreateData($original, $form, $storeId);
// echo "<pre>";
// print_r($data);
// exit();
// 事务处理:添加商品
$model = new GoodsModel();
$model->transaction(function () use ($model, $data, $storeId) {
// 添加商品
$model->save($data);
$model->where('goods_id', (int)$model['goods_id'])->update(['spu_id' => (int)$model['goods_id']]);
// 新增商品与分类关联
GoodsCategoryRelModel::increased((int)$model['goods_id'], $data['categoryIds'], $storeId);
// 新增商品与图片关联
GoodsImageModel::increased((int)$model['goods_id'], $data['imagesIds'], $storeId);
// 新增商品与规格关联
GoodsSpecRelModel::increased((int)$model['goods_id'], $data['newSpecList'], $storeId);
// 新增商品sku信息
GoodsSkuModel::add((int)$model['goods_id'], $data['spec_type'], $data['newSkuList'], $storeId);
});
return true;
}
/**
* 后台单个采集
* [single description]
* @param string $url [description]
* @param array $form [description]
* @param int $storeId [description]
* @return [type] [description]
*/
public function updateGoods(string $url, array $form, int $storeId): bool
{
$goods = GoodsModel::where('goods_id', $form['goods_id'])->find();
//商品不存在
if (!$goods || !$url) {
return false;
}
//新阙通信的抓取过了,就不抓取了
if ($goods['channel'] == 'xqtx' && $goods['content']) {
return true;
}
try {
// 采集第三方商品数据
$original = $this->collector($url, $storeId);
// echo "<pre>";
// print_r($original);
// exit();
if ($original['spec_type'] == 20) {
$original['spec_type'] = 10;
$skuList = array_column($original['specData']['skuList'], null, "goods_sku_no");
$goods_price = $skuList[$original['goods_sku_no']]['goods_price'] ?? 0;
$original['goods_price'] = $goods_price;
$original['line_price'] = $goods_price;
$original['data_type'] = 1;//苏宁的数据变成的人工数据了,不再使用苏宁接口更新数据
$original['link_other'] = $url;
unset($original['specData']);
}
$form['imageStorage'] = 10;
// 下载远程商品图片
$original = $this->thirdPartyImages($original, $form['imageStorage'], $storeId);
} catch (\Throwable $e) {
// var_dump($e->getMessage());
// exit;
tre($e->getTraceAsString());
$this->errorLog[] = ['url' => trim($url), 'message' => $e->getMessage()];
return false;
}
if ($original['goods_price'] == 0) {
return false;
}
$original['cost_price'] = $form['cost_price_min'];
$data['link_other'] = $url;
$data['goods_no'] = $original['goods_sku_no'];
$data['content'] = $original['content'];
//重新计算利润和利润率
$data['goods_price_min'] = $original['goods_price'];
$data['goods_price_max'] = $original['goods_price'];
$data['line_price_min'] = $original['goods_price'];
$data['line_price_max'] = $original['goods_price'];
$data['profit'] = $original['goods_price'] - $original['cost_price'];
$profit_rate = (float)$original['goods_price'] > 0 ? ($original['goods_price'] - $original['cost_price']) / $original['goods_price'] : 0.00;
$profit_rate = $profit_rate > 0.0001 ? bcmul((string)$profit_rate, "100", 2) : 0.00;
$data['profit_rate'] = $profit_rate;
// echo "<pre>";
// print_r($data);
// exit();
// 事务处理:添加商品
$model = new GoodsModel();
$goodsSku = [
'goods_price' => $original['goods_price'],
'goods_sku_no' => $original['goods_sku_no'],
];
//新阙通信的数据
if ($goods->channel == "xqtx") {
//goods
$data['stock_total'] = $form['stock_total'];
$data['goods_name'] = $original['goods_name'];
//设置当前商品价格
$data['profit'] = $original['goods_price'] - $original['cost_price'];
$profit_rate = (float)$original['goods_price'] > 0 ? ($original['goods_price'] - $original['cost_price']) / $original['goods_price'] : 0.00;
$profit_rate = $profit_rate > 0.0001 ? bcmul((string)$profit_rate, "100", 2) : 0.00;
$data['profit_rate'] = $profit_rate;
unset($data['link_other']);
//sku
$goodsSku['stock_num'] = $form['stock_total'];
unset($goodsSku['goods_price']);
}
// var_dump($data);
// var_dump($goodsSku);
// exit();
$model->transaction(function () use ($form, $data, $original, $goodsSku) {
// 添加商品
GoodsModel::where('goods_id', $form['goods_id'])->update($data);
// 新增商品与图片关联
GoodsImageModel1::updates((int)$form['goods_id'], $original['imagesIds']);
//更新sku信息
GoodsSkuModel::where('goods_id', $form['goods_id'])->update($goodsSku);
});
return true;
}
/**
* 检查采集记录状态是否异常
* @param int $recordId
* @return bool
*/
private function checkStatusFail(int $recordId): bool
{
$model = $this->getRecord($recordId);
return $model['status'] == GoodsCollectorStatusEnum::FAIL || $model['is_delete'];
}
/**
* 抓取第三方商品内容
* @param string $url
* @param int $storeId
* @return array
* @throws BaseException
* @throws \think\db\exception\DataNotFoundException
* @throws \think\db\exception\DbException
* @throws \think\db\exception\ModelNotFoundException
*/
public function collector(string $url, int $storeId): array
{
// 获取商品来源
$store = $this->getStore($url);
// 获取url中的商品ID
$itemId = $this->getItemId($url, $store);
// 商城采集设置
$config = SettingModel::getItem(SettingEnum::COLLECTOR, $storeId);
if (isset($config['config']['99api']['apiKey']) && !$config['config']['99api']['apiKey']) {
$config['config']['99api']['apiKey'] = '9271DA7BF37C6F50BED49B7F0ADD49A2';
}
// 请求API查询商品详情
$item = CollectorFacade::store($store)
->setOptions($config['config'][$config['provider']])
->detail($itemId);
$item['goods_sku_no'] = $itemId;
return $item;
}
/**
* 根据url获取商品来源
* @param string $url
* @return string
* @throws BaseException
*/
private function getStore(string $url): string
{
$UrlStore = new UrlStore($url);
if (!$store = $UrlStore->getStore()) {
throwError('不支持该网址的采集,请检查url是否正确');
}
return $store;
}
/**
* 获取url中的商品ID
* @param string $url
* @param string $store
* @return string
* @throws BaseException
*/
private function getItemId(string $url, string $store): string
{
$UrlItemId = new UrlItemId($url, $store);
if (!$itemId = $UrlItemId->getItemId()) {
throwError('未能获取到商品ID,请检查url是否正确');
}
return $itemId;
}
/**
* 更新采集记录
* @param int $recordId 商品采集记录ID
* @param int $currentCount 当前任务采集的商品总量
* @return void
*/
private function updateRecord(int $recordId, int $currentCount)
{
// 获取采集记录
$model = $this->getRecord($recordId);
// 计算采集失败的数量
$failCount = $currentCount - $this->successCount;
// 更新采集记录
$model->save([
'success_count' => $model['success_count'] + $this->successCount,
'fail_count' => $model['fail_count'] + $failCount,
'fail_log' => array_merge($model['fail_log'], $this->errorLog),
]);
// 判断是否为最后一次队列
if ($model['total_count'] <= ($model['success_count'] + $model['fail_count'])) {
$model->save(['end_time' => \time(), 'status' => GoodsCollectorStatusEnum::COMPLETED]);
}
}
/**
* 获取采集记录
* @param int $recordId 商品采集记录ID
* @return GoodsCollectorModel|array|null
*/
private function getRecord(int $recordId)
{
if (!$this->record) {
$this->record = GoodsCollectorModel::detail($recordId);
}
return $this->record;
}
/**
* 生成商品数据(用于写入数据库)
* @param array $original 商品原始数据
* @param array $form 用户提交的表单
* @param int $storeId 当前商城id
* @return array
* @throws BaseException
* @throws \think\db\exception\DataNotFoundException
* @throws \think\db\exception\DbException
* @throws \think\db\exception\ModelNotFoundException
*/
private function singleCreateData(array $original, array $form, int $storeId): array
{
// 整理商品数据
$data = [
'goods_type' => $form['goods_type'],
'delivery_id' => $form['delivery_id'] ?? 0,
'goods_name' => $original['goods_name'],
'goods_no' => $original['goods_sku_no'] ?? "",
'data_type' => $original['data_type'],//数据类型
'link' => $original['link'],//采集地址
'spec_type' => $original['spec_type'],
'unicode' => $original['spec_type'],
'delivery_id' => $form['delivery_id'] ?? 0,
'content' => $original['content'] ?? '',
'sort' => 100,
'deduct_stock_type' => DeductStockTypeEnum::CREATE,
'status' => $form['goods_status'],
'imagesIds' => $original['imagesIds'],
'categoryIds' => $form['categoryIds'],
// 下面是默认数据, 没有会报错
'alone_grade_equity' => [],
'newSpecList' => [],
'newSkuList' => [],
'store_id' => $storeId,
];
if (isset($form['channel'])) {
$data['channel'] = $form['channel'];
}
$data['unicode'] = $data['channel']."-0-".$original['goods_sku_no'];
// 整理商品的价格和库存总量
if ($data['spec_type'] === GoodsSpecTypeEnum::MULTI) {
$data['stock_total'] = GoodsSkuModel::getStockTotal($original['specData']['skuList']);
[$data['goods_price_min'], $data['goods_price_max']] = GoodsSkuModel::getGoodsPrices($original['specData']['skuList']);
[$data['line_price_min'], $data['line_price_max']] = GoodsSkuModel::getLinePrices($original['specData']['skuList']);
} elseif ($data['spec_type'] === GoodsSpecTypeEnum::SINGLE) {
$data['goods_price_min'] = $data['goods_price_max'] = $original['goods_price'];
$data['line_price_min'] = $data['line_price_max'] = $original['goods_price'];
$data['cost_price_min'] = $original['cost_price'] ?? 0.00;
$data['profit'] = (float)$original['goods_price'] - (float)$original['cost_price'];
$profit_rate = (float)$original['goods_price'] > 0 ? ((float)$original['goods_price'] - (float)$original['cost_price']) / (float)$original['goods_price'] : 0.00;
$profit_rate = $profit_rate > 0.0001 ? bcmul((string)$profit_rate, "100", 2) : 0.00;
$data['profit_rate'] = $profit_rate;
$data['stock_total'] = $original['stock_num'];
$data['remark'] = $original['remark'] ?? "";
$data['cmmdty_model'] = $original['cmmdty_model'] ?? "";
$data['goods_no'] = $original['goods_no'] ?? "";
$data['region'] = $original['region'] ?? "";
$data['region_text'] = $original['region_text'] ?? "";
}
// 规格和sku数据处理
if ($data['spec_type'] === GoodsSpecTypeEnum::MULTI) {
// 验证规格值是否合法
SpecModel::checkSpecData($original['specData']['specList']);
// 生成多规格数据 (携带id)
$data['newSpecList'] = SpecModel::getNewSpecList($original['specData']['specList'], $storeId);
// 生成skuList (携带goods_sku_id)
$data['newSkuList'] = GoodsSkuModel::getNewSkuList($data['newSpecList'], $original['specData']['skuList']);
} elseif ($data['spec_type'] === GoodsSpecTypeEnum::SINGLE) {
// 生成skuItem
$data['newSkuList'] = helper::pick($original, ['goods_sku_no','goods_price', 'line_price', 'cost_price','stock_num', 'goods_weight']);
}
return $data;
}
/**
* 生成商品数据(用于写入数据库)
* @param array $original 商品原始数据
* @param array $form 用户提交的表单
* @param int $storeId 当前商城id
* @return array
* @throws BaseException
* @throws \think\db\exception\DataNotFoundException
* @throws \think\db\exception\DbException
* @throws \think\db\exception\ModelNotFoundException
*/
private function createData(array $original, array $form, int $storeId): array
{
// 整理商品数据
$data = [
'goods_type' => $form['goods_type'],
'goods_name' => $original['goods_name'],
'goods_no' => $original['goods_sku_no'] ?? "",
'data_type' => $original['data_type'],//数据类型
'link' => $original['link'],//采集地址
'spec_type' => $original['spec_type'],
'delivery_id' => $form['delivery_id'] ?? 0,
'content' => $original['content'] ?? '',
'sort' => 100,
'deduct_stock_type' => DeductStockTypeEnum::CREATE,
'status' => $form['goods_status'],
'imagesIds' => $original['imagesIds'],
'categoryIds' => $form['categoryIds'],
// 下面是默认数据, 没有会报错
'alone_grade_equity' => [],
'newSpecList' => [],
'newSkuList' => [],
'store_id' => $storeId,
];
if (isset($form['channel'])) {
$data['channel'] = $form['channel'];
}
// 整理商品的价格和库存总量
if ($data['spec_type'] === GoodsSpecTypeEnum::MULTI) {
$data['stock_total'] = GoodsSkuModel::getStockTotal($original['specData']['skuList']);
[$data['goods_price_min'], $data['goods_price_max']] = GoodsSkuModel::getGoodsPrices($original['specData']['skuList']);
[$data['line_price_min'], $data['line_price_max']] = GoodsSkuModel::getLinePrices($original['specData']['skuList']);
} elseif ($data['spec_type'] === GoodsSpecTypeEnum::SINGLE) {
$data['goods_price_min'] = $data['goods_price_max'] = $original['goods_price'];
$data['line_price_min'] = $data['line_price_max'] = $original['line_price'];
$data['line_price_min'] = $data['line_price_max'] = $original['line_price'];
$data['stock_total'] = $original['stock_num'] ?? 100;
}
// 规格和sku数据处理
if ($data['spec_type'] === GoodsSpecTypeEnum::MULTI) {
// 验证规格值是否合法
SpecModel::checkSpecData($original['specData']['specList']);
// 生成多规格数据 (携带id)
$data['newSpecList'] = SpecModel::getNewSpecList($original['specData']['specList'], $storeId);
// 生成skuList (携带goods_sku_id)
$data['newSkuList'] = GoodsSkuModel::getNewSkuList($data['newSpecList'], $original['specData']['skuList']);
} elseif ($data['spec_type'] === GoodsSpecTypeEnum::SINGLE) {
// 生成skuItem
$data['newSkuList'] = helper::pick($original, ['goods_sku_no','goods_price', 'line_price', 'stock_num', 'goods_weight']);
}
return $data;
}
/**
* 下载远程商品图片
* @param array $original 商品信息
* @param int $imageStorage 商品主图 (10下载到本地、20使用源图片url)
* @param int $storeId
* @return array
* @throws BaseException
* @throws \think\Exception
* @throws \think\db\exception\DataNotFoundException
* @throws \think\db\exception\DbException
* @throws \think\db\exception\ModelNotFoundException
*/
private function thirdPartyImages(array $original, int $imageStorage, int $storeId): array
{
// 保存商品主图
$original['imagesIds'] = [];
foreach ($original['goodsImages'] as $imageUrl) {
$original['imagesIds'][] = $imageStorage === 10 ? $this->downImage($imageUrl, $storeId)
: $this->externalImage($imageUrl, $storeId);
}
// 保存商品SKU封面图
if ($original['spec_type'] === GoodsSpecTypeEnum::MULTI) {
foreach ($original['specData']['skuList'] as &$skuItem) {
if (!empty($skuItem['imageUrl'])) {
$skuItem['image_id'] = $imageStorage === 10 ? $this->downImage($skuItem['imageUrl'], $storeId)
: $this->externalImage($skuItem['imageUrl'], $storeId);
}
}
}
return $original;
}
/**
* 将远程图片外链保存到文件库中
* @param string $imageUrl 远程图片url
* @param int $storeId
* @return mixed
* @throws BaseException
*/
private function externalImage(string $imageUrl, int $storeId)
{
// 如果已存在该文件则直接返回文件ID
if (isset($this->dictImageUrls[$imageUrl])) {
return $this->dictImageUrls[$imageUrl];
}
// 文件外链记录到文件库
$UploadService = (new UploadService)->setStoreId($storeId);
if (!$UploadService->uploadByExternal(FileTypeEnum::IMAGE, $imageUrl)) {
throwError('图片记录失败:' . $UploadService->getError());
}
// 记录文件ID
$fileId = $UploadService->getFileInfo()['file_id'];
$this->dictImageUrls[$imageUrl] = $fileId;
return $fileId;
}
/**
* 将远程图片下载并保存到文件库中
* @param string $imageUrl 远程图片url
* @param int $storeId
* @return mixed
* @throws BaseException
* @throws \think\Exception
* @throws \think\db\exception\DataNotFoundException
* @throws \think\db\exception\DbException
* @throws \think\db\exception\ModelNotFoundException
*/
private function downImage(string $imageUrl, int $storeId)
{
// 如果已存在该文件则直接返回文件ID
if (isset($this->dictImageUrls[$imageUrl])) {
return $this->dictImageUrls[$imageUrl];
}
// 下载远程图片
$filePath = (new Download)->saveTempImage($storeId, $imageUrl);
// 文件上传到文件库
$UploadService = (new UploadService)->setStoreId($storeId);
if (!$UploadService->uploadByLocal(FileTypeEnum::IMAGE, $filePath)) {
throwError('图片上传失败:' . $UploadService->getError());
}
// 记录文件ID
$fileId = $UploadService->getFileInfo()['file_id'];
$this->dictImageUrls[$imageUrl] = $fileId;
return $fileId;
}
}