Initial Upload
This commit is contained in:
Matt Batchelder
280
lib/XTR/RemoteDataSetFetchTask.php
Normal file
280
lib/XTR/RemoteDataSetFetchTask.php
Normal file
@@ -0,0 +1,280 @@
|
||||
<?php
|
||||
/*
|
||||
* Copyright (C) 2023 Xibo Signage Ltd
|
||||
*
|
||||
* Xibo - Digital Signage - https://xibosignage.com
|
||||
*
|
||||
* This file is part of Xibo.
|
||||
*
|
||||
* Xibo is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Xibo is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with Xibo. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
|
||||
namespace Xibo\XTR;
|
||||
|
||||
use Carbon\Carbon;
|
||||
use Xibo\Entity\DataSet;
|
||||
use Xibo\Factory\DataSetFactory;
|
||||
use Xibo\Factory\NotificationFactory;
|
||||
use Xibo\Factory\UserFactory;
|
||||
use Xibo\Factory\UserGroupFactory;
|
||||
use Xibo\Support\Exception\GeneralException;
|
||||
|
||||
/**
|
||||
* Class RemoteDataSetFetchTask
|
||||
* @package Xibo\XTR
|
||||
*/
|
||||
class RemoteDataSetFetchTask implements TaskInterface
|
||||
{
|
||||
use TaskTrait;
|
||||
|
||||
/** @var DataSetFactory */
|
||||
private $dataSetFactory;
|
||||
|
||||
/** @var NotificationFactory */
|
||||
private $notificationFactory;
|
||||
|
||||
/** @var UserFactory */
|
||||
private $userFactory;
|
||||
|
||||
/** @var UserGroupFactory */
|
||||
private $userGroupFactory;
|
||||
|
||||
/** @inheritdoc */
|
||||
public function setFactories($container)
|
||||
{
|
||||
$this->dataSetFactory = $container->get('dataSetFactory');
|
||||
$this->notificationFactory = $container->get('notificationFactory');
|
||||
$this->userFactory = $container->get('userFactory');
|
||||
$this->userGroupFactory = $container->get('userGroupFactory');
|
||||
return $this;
|
||||
}
|
||||
|
||||
/**
|
||||
* @inheritdoc
|
||||
*/
|
||||
public function run()
|
||||
{
|
||||
$this->runMessage = '# ' . __('Fetching Remote-DataSets') . PHP_EOL . PHP_EOL;
|
||||
|
||||
$runTime = Carbon::now()->format('U');
|
||||
|
||||
/** @var DataSet $dataSet */
|
||||
$dataSet = null;
|
||||
|
||||
// Process all Remote DataSets (and their dependants)
|
||||
$dataSets = $this->orderDataSetsByDependency($this->dataSetFactory->query(null, ['isRemote' => 1]));
|
||||
|
||||
// Log the order.
|
||||
$this->log->debug(
|
||||
'Order of processing: ' . json_encode(array_map(function ($element) {
|
||||
return $element->dataSetId . ' - ' . $element->runsAfter;
|
||||
}, $dataSets))
|
||||
);
|
||||
|
||||
// Reorder this list according to which order we want to run in
|
||||
foreach ($dataSets as $dataSet) {
|
||||
$this->log->debug('Processing ' . $dataSet->dataSet . '. ID:' . $dataSet->dataSetId);
|
||||
$hardRowLimit = $this->config->getSetting('DATASET_HARD_ROW_LIMIT');
|
||||
$softRowLimit = $dataSet->rowLimit;
|
||||
$limitPolicy = $dataSet->limitPolicy;
|
||||
$currentNumberOfRows = intval($this->store->select('SELECT COUNT(*) AS total FROM `dataset_' . $dataSet->dataSetId . '`', [])[0]['total']);
|
||||
|
||||
try {
|
||||
// Has this dataSet been accessed recently?
|
||||
if (!$dataSet->isActive()) {
|
||||
// Skipping dataSet due to it not being accessed recently
|
||||
$this->log->info('Skipping dataSet ' . $dataSet->dataSetId . ' due to it not being accessed recently');
|
||||
continue;
|
||||
}
|
||||
|
||||
// Get all columns
|
||||
$columns = $dataSet->getColumn();
|
||||
|
||||
// Filter columns where dataSetColumnType is "Remote"
|
||||
$filteredColumns = array_filter($columns, function ($column) {
|
||||
return $column->dataSetColumnTypeId == '3';
|
||||
});
|
||||
|
||||
// Check if there are any remote columns defined in the dataset
|
||||
if (count($filteredColumns) === 0) {
|
||||
$this->log->info('Skipping dataSet ' . $dataSet->dataSetId . ': No remote columns defined in the dataset.');
|
||||
continue;
|
||||
}
|
||||
|
||||
$this->log->debug('Comparing run time ' . $runTime . ' to next sync time ' . $dataSet->getNextSyncTime());
|
||||
|
||||
if ($runTime >= $dataSet->getNextSyncTime()) {
|
||||
// Getting the dependant DataSet to process the current DataSet on
|
||||
$dependant = null;
|
||||
if ($dataSet->runsAfter != null && $dataSet->runsAfter != $dataSet->dataSetId) {
|
||||
$dependant = $this->dataSetFactory->getById($dataSet->runsAfter);
|
||||
}
|
||||
|
||||
$this->log->debug('Fetch and process ' . $dataSet->dataSet);
|
||||
|
||||
$results = $this->dataSetFactory->callRemoteService($dataSet, $dependant);
|
||||
|
||||
if ($results->number > 0) {
|
||||
// Truncate only if we also fetch new Data
|
||||
if ($dataSet->isTruncateEnabled()
|
||||
&& $results->isEligibleToTruncate
|
||||
&& $runTime >= $dataSet->getNextClearTime()
|
||||
) {
|
||||
$this->log->debug('Truncate ' . $dataSet->dataSet);
|
||||
$dataSet->deleteData();
|
||||
|
||||
// Update the last clear time.
|
||||
$dataSet->saveLastClear($runTime);
|
||||
}
|
||||
|
||||
$rowsToAdd = $results->number;
|
||||
$this->log->debug('Current number of rows in DataSet ID ' . $dataSet->dataSetId . ' is: ' . $currentNumberOfRows . ' number of records to add ' . $rowsToAdd);
|
||||
|
||||
// row limit reached
|
||||
if ($currentNumberOfRows + $rowsToAdd >= $hardRowLimit || $softRowLimit != null && $currentNumberOfRows + $rowsToAdd >= $softRowLimit) {
|
||||
// handle remote DataSets created before introduction of limit policy
|
||||
if ($limitPolicy == null) {
|
||||
$this->log->debug('No limit policy set, default to stop syncing.');
|
||||
$limitPolicy = 'stop';
|
||||
}
|
||||
|
||||
// which limit policy was set?
|
||||
if ($limitPolicy === 'stop') {
|
||||
$this->log->info('DataSet ID ' . $dataSet->dataSetId . ' reached the row limit, due to selected limit policy, it will stop syncing');
|
||||
continue;
|
||||
} elseif ($limitPolicy === 'fifo') {
|
||||
// FiFo
|
||||
$this->log->info('DataSet ID ' . $dataSet->dataSetId . ' reached the row limit, due to selected limit policy, oldest rows will be removed');
|
||||
|
||||
$this->store->update('DELETE FROM `dataset_' . $dataSet->dataSetId . '` ORDER BY id ASC LIMIT ' . $rowsToAdd, []);
|
||||
} elseif ($limitPolicy === 'truncate') {
|
||||
// truncate
|
||||
$this->log->info('DataSet ID ' . $dataSet->dataSetId . ' reached the row limit, due to selected limit policy, we will truncate the DataSet data');
|
||||
$dataSet->deleteData();
|
||||
|
||||
// Update the last clear time.
|
||||
$dataSet->saveLastClear($runTime);
|
||||
}
|
||||
}
|
||||
|
||||
if ($dataSet->sourceId === 1) {
|
||||
$this->dataSetFactory->processResults($dataSet, $results);
|
||||
} else {
|
||||
$this->dataSetFactory->processCsvEntries($dataSet, $results);
|
||||
}
|
||||
|
||||
// notify here
|
||||
$dataSet->notify();
|
||||
} else if ($dataSet->truncateOnEmpty
|
||||
&& $results->isEligibleToTruncate
|
||||
&& $dataSet->isTruncateEnabled()
|
||||
&& $runTime >= $dataSet->getNextClearTime()
|
||||
) {
|
||||
$this->log->debug('Truncate ' . $dataSet->dataSet);
|
||||
$dataSet->deleteData();
|
||||
|
||||
// Update the last clear time.
|
||||
$dataSet->saveLastClear($runTime);
|
||||
$this->appendRunMessage(__('No results for %s, truncate with no new data enabled', $dataSet->dataSet));
|
||||
} else {
|
||||
$this->appendRunMessage(__('No results for %s', $dataSet->dataSet));
|
||||
}
|
||||
|
||||
$dataSet->saveLastSync($runTime);
|
||||
} else {
|
||||
$this->log->debug('Sync not required for ' . $dataSet->dataSetId);
|
||||
}
|
||||
} catch (GeneralException $e) {
|
||||
$this->appendRunMessage(__('Error syncing DataSet %s', $dataSet->dataSet));
|
||||
$this->log->error('Error syncing DataSet ' . $dataSet->dataSetId . '. E = ' . $e->getMessage());
|
||||
$this->log->debug($e->getTraceAsString());
|
||||
|
||||
// Send a notification to the dataSet owner, informing them of the failure.
|
||||
$notification = $this->notificationFactory->createEmpty();
|
||||
$notification->subject = __('Remote DataSet %s failed to synchronise', $dataSet->dataSet);
|
||||
$notification->body = 'The error is: ' . $e->getMessage();
|
||||
$notification->createDt = Carbon::now()->format('U');
|
||||
$notification->releaseDt = $notification->createDt;
|
||||
$notification->isInterrupt = 0;
|
||||
$notification->userId = $this->user->userId;
|
||||
$notification->type = 'dataset';
|
||||
|
||||
// Assign me
|
||||
$dataSetUser = $this->userFactory->getById($dataSet->userId);
|
||||
$notification->assignUserGroup($this->userGroupFactory->getById($dataSetUser->groupId));
|
||||
|
||||
// Send
|
||||
$notification->save();
|
||||
|
||||
// You might say at this point that if there are other data sets further down the list, we shouldn't
|
||||
// continue because they might depend directly on this one
|
||||
// however, it is my opinion that they should be processed anyway with the current cache of data.
|
||||
// hence continue
|
||||
}
|
||||
}
|
||||
|
||||
$this->appendRunMessage(__('Done'));
|
||||
}
|
||||
|
||||
/**
|
||||
* Order the list of DataSets to be processed so that it is dependent aware.
|
||||
*
|
||||
* @param DataSet[] $dataSets Reference to an Array which holds all not yet processed DataSets
|
||||
* @return DataSet[] Ordered list of DataSets to process
|
||||
*
|
||||
*
|
||||
* What is going on here: RemoteDataSets can depend on others, so we have to be sure to fetch
|
||||
* the data from the dependant first.
|
||||
* For Example (id, dependant): (1,4), (2,3), (3,4), (4,1), (5,2), (6,6)
|
||||
* Should be processed like: 4, 1, 3, 2, 5, 6
|
||||
*
|
||||
*/
|
||||
private function orderDataSetsByDependency(array $dataSets)
|
||||
{
|
||||
// DataSets are in no particular order
|
||||
// sort them according to their dependencies
|
||||
usort($dataSets, function ($a, $b) {
|
||||
/** @var DataSet $a */
|
||||
/** @var DataSet $b */
|
||||
// if a doesn't have a dependent, then a must be lower in the list (move b up)
|
||||
if ($a->runsAfter === null) {
|
||||
return -1;
|
||||
}
|
||||
// if b doesn't have a dependent, then a must be higher in the list (move b down)
|
||||
if ($b->runsAfter === null) {
|
||||
return 1;
|
||||
}
|
||||
// either a or b have a dependent
|
||||
// if they are the same, keep them where they are
|
||||
if ($a->runsAfter === $b->runsAfter) {
|
||||
return 0;
|
||||
}
|
||||
// the dependents are different.
|
||||
// if a depends on b, then move b up
|
||||
if ($a->runsAfter === $b->dataSetId) {
|
||||
return -1;
|
||||
}
|
||||
// if b depends on a, then move b down
|
||||
if ($b->runsAfter === $a->dataSetId) {
|
||||
return 1;
|
||||
}
|
||||
// Unsorted
|
||||
return 0;
|
||||
});
|
||||
|
||||
// Process in reverse order (LastIn-FirstOut)
|
||||
return array_reverse($dataSets);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user