Loading Classes/WE/SpreadsheetImport/Command/SpreadsheetImportCommandController.php +49 −19 Original line number Diff line number Diff line Loading @@ -51,7 +51,7 @@ class SpreadsheetImportCommandController extends CommandController { public function importCommand() { $currentImportingCount = $this->spreadsheetImportRepository->countByImportingStatus(SpreadsheetImport::IMPORTING_STATUS_IN_PROGRESS); if ($currentImportingCount > 0) { $this->outputFormatted('Previous spreadsheet import is still in progress.'); $this->outputLine('Previous spreadsheet import is still in progress.'); $this->quit(); } /** @var SpreadsheetImport $spreadsheetImport */ Loading @@ -67,35 +67,65 @@ class SpreadsheetImportCommandController extends CommandController { try { $this->spreadsheetImportService->import(); $spreadsheetImport->setImportingStatus(SpreadsheetImport::IMPORTING_STATUS_COMPLETED); $this->outputFormatted('Spreadsheet has been imported. %d inserted, %d updated, %d deleted, %d skipped', $this->outputLine('Spreadsheet has been imported. %d inserted, %d updated, %d deleted, %d skipped', array($spreadsheetImport->getTotalInserted(), $spreadsheetImport->getTotalUpdated(), $spreadsheetImport->getTotalDeleted(), $spreadsheetImport->getTotalSkipped())); } catch (Exception $e) { } catch (\Exception $e) { $spreadsheetImport->setImportingStatus(SpreadsheetImport::IMPORTING_STATUS_FAILED); $this->outputFormatted('Spreadsheet import failed.'); $this->outputLine('Spreadsheet import failed.'); } try { $this->spreadsheetImportRepository->update($spreadsheetImport); } catch (\Exception $e) { $this->outputLine('Spreadsheet import status update error. It remains in progress until cleanup.'); } } else { $this->outputFormatted('No spreadsheet import in queue.'); } } /** * Cleanup previous spreadsheet imports. Threashold defined in settings. * Cleanup past and stalled imports. * * @param int $keepPastImportsThreasholdDays Overwrites the setting value * @param int $maxExecutionThreasholdMinutes Overwrites the setting value */ public function cleanupCommand() { $cleanupImportsThreasholdDays = intval($this->settings['cleanupImportsThreasholdDays']); $cleanupFromDate = new \DateTime(); $cleanupFromDate->sub(new \DateInterval('P' . $cleanupImportsThreasholdDays . 'D')); $oldSpreadsheetImports = $this->spreadsheetImportRepository->findPreviousImportsBySpecificDate($cleanupFromDate); if ($oldSpreadsheetImports->count() > 0) { /** @var SpreadsheetImport $oldSpreadsheetImport */ foreach ($oldSpreadsheetImports as $oldSpreadsheetImport) { $this->spreadsheetImportRepository->remove($oldSpreadsheetImport); public function cleanupCommand($keepPastImportsThreasholdDays = -1, $maxExecutionThreasholdMinutes = -1) { $keepPastImportsThreasholdDays = ($keepPastImportsThreasholdDays >= 0) ? $keepPastImportsThreasholdDays : intval($this->settings['keepPastImportsThreasholdDays']); $maxExecutionThreasholdMinutes = ($maxExecutionThreasholdMinutes >= 0) ? $maxExecutionThreasholdMinutes : intval($this->settings['maxExecutionThreasholdMinutes']); $this->cleanupPastImports($keepPastImportsThreasholdDays); $this->cleanupStalledImports($maxExecutionThreasholdMinutes); } $this->outputLine('%d spreadsheet imports were removed.', array($oldSpreadsheetImports->count())); } else { $this->outputLine('There is no spreadsheet import in queue to remove.'); /** * Delete past imports * * @param int $keepPastImportsThreasholdDays */ private function cleanupPastImports($keepPastImportsThreasholdDays) { $cleanupFromDateTime = new \DateTime(); $cleanupFromDateTime->sub(new \DateInterval('P' . $keepPastImportsThreasholdDays . 'D')); $spreadsheetImports = $this->spreadsheetImportRepository->findBySpecificDateTimeAndImportingStatus($cleanupFromDateTime); /** @var SpreadsheetImport $spreadsheetImport */ foreach ($spreadsheetImports as $spreadsheetImport) { $this->spreadsheetImportRepository->remove($spreadsheetImport); } $this->outputLine('%d spreadsheet imports removed.', array($spreadsheetImports->count())); } /** * Set stalled imports to failed * * @param int $maxExecutionThreasholdMinutes */ private function cleanupStalledImports($maxExecutionThreasholdMinutes) { $cleanupFromDateTime = new \DateTime(); $cleanupFromDateTime->sub(new \DateInterval('PT' . $maxExecutionThreasholdMinutes . 'M')); $spreadsheetImports = $this->spreadsheetImportRepository->findBySpecificDateTimeAndImportingStatus($cleanupFromDateTime, SpreadsheetImport::IMPORTING_STATUS_IN_PROGRESS); /** @var SpreadsheetImport $spreadsheetImport */ foreach ($spreadsheetImports as $spreadsheetImport) { $spreadsheetImport->setImportingStatus(SpreadsheetImport::IMPORTING_STATUS_FAILED); $this->spreadsheetImportRepository->update($spreadsheetImport); } $this->outputLine('%d spreadsheet imports set to failed.', array($spreadsheetImports->count())); } } Classes/WE/SpreadsheetImport/Domain/Model/SpreadsheetImport.php +19 −1 Original line number Diff line number Diff line Loading @@ -38,10 +38,19 @@ class SpreadsheetImport { protected $file; /** * DateTime when import is scheduled to progress * * @var \DateTime */ protected $scheduleDate; /** * Actual DateTime when import is set to in progress * * @var \DateTime */ protected $progressDate; /** * @var string * @ORM\Column(type="text") Loading Loading @@ -100,6 +109,7 @@ class SpreadsheetImport { */ public function __construct() { $this->scheduleDate = new \DateTime(); $this->progressDate = new \DateTime(); } /** Loading Loading @@ -142,6 +152,7 @@ class SpreadsheetImport { */ public function setScheduleDate($scheduleDate) { $this->scheduleDate = $scheduleDate; $this->progressDate = $scheduleDate; } /** Loading @@ -166,7 +177,11 @@ class SpreadsheetImport { * @return array */ public function getArguments() { return unserialize($this->arguments); $arguments = unserialize($this->arguments); if (! is_array($arguments)) { $arguments = array(); } return $arguments; } /** Loading Loading @@ -234,6 +249,9 @@ class SpreadsheetImport { */ public function setImportingStatus($importingStatus) { $this->importingStatus = $importingStatus; if ($importingStatus === self::IMPORTING_STATUS_IN_PROGRESS) { $this->progressDate = new \DateTime(); } } /** Loading Classes/WE/SpreadsheetImport/Domain/Repository/SpreadsheetImportRepository.php +6 −2 Original line number Diff line number Diff line Loading @@ -42,12 +42,16 @@ class SpreadsheetImportRepository extends Repository { /** * @param \DateTime $dateTime * @param int $importingStatus * * @return \TYPO3\Flow\Persistence\QueryResultInterface */ public function findPreviousImportsBySpecificDate(\DateTime $dateTime) { public function findBySpecificDateTimeAndImportingStatus(\DateTime $dateTime, $importingStatus = -1) { $query = $this->createQuery(); $constraint = $query->lessThanOrEqual('scheduleDate', $dateTime); $constraint = $query->lessThanOrEqual('progressDate', $dateTime); if ($importingStatus >= 0) { $constraint = $query->logicalAnd($constraint, $query->equals('importingStatus', $importingStatus)); } return $query->matching($constraint)->execute(); } Loading Classes/WE/SpreadsheetImport/FrontendMappingService.php +1 −1 Original line number Diff line number Diff line Loading @@ -122,7 +122,7 @@ class FrontendMappingService { $previewObject = $this->spreadsheetImportService->getObjectByRow($record); $preview = array(); $hasErrors = FALSE; $objectValidator = $this->validatorResolver->getBaseValidatorConjunction($domain); $objectValidator = $this->validatorResolver->getBaseValidatorConjunction($domain, SpreadsheetImportService::VALIDATION_GROUPS); $errors = $objectValidator->validate($previewObject)->getFlattenedErrors(); foreach ($mapping as $property => $columnMapping) { /** @var Mapping $mapping */ Loading Classes/WE/SpreadsheetImport/SpreadsheetImportService.php +174 −102 Original line number Diff line number Diff line Loading @@ -12,22 +12,29 @@ namespace WE\SpreadsheetImport; * */ use TYPO3\Flow\Annotations as Flow; use TYPO3\Flow\Persistence\QueryInterface; use TYPO3\Flow\Persistence\RepositoryInterface; use WE\SpreadsheetImport\Annotations\Mapping; use WE\SpreadsheetImport\Domain\Model\SpreadsheetImport; /** * Service to handle the column mapping and the import itself * * @Flow\Scope("singleton") */ class SpreadsheetImportService { const VALIDATION_GROUPS = array('Default', 'SpreadsheetImport'); /** * Domain object containing the configuration the service works with * * @var SpreadsheetImport */ protected $spreadsheetImport; /** * Class name of the domain model to be imported * * @var string */ protected $domain; Loading Loading @@ -69,25 +76,59 @@ class SpreadsheetImportService { protected $validatorResolver; /** * Inverse SpreadsheetImport mapping array * Inverse array of SpreadsheetDomain mapping array property. Always use the getter function, which will process the * property in case it is not set. * * @internal * @see SpreadsheetImportService::getInverseMappingProperties() * @var array */ private $inverseMappingProperties; /** * Identifier properties of SpreadsheetDomain mapping array. Always use the getter function, which will process the * property in case it is not set. * * @internal * @see SpreadsheetImportService::getMappingIdentifierProperties() * @var array */ private $inverseSpreadsheetImportMapping; private $mappingIdentifierProperties; /** * Identifier properties of SpreadsheetDomain argument array. Always use the getter function, which will process the * property in case it is not set. * * @internal * @see SpreadsheetImportService::getArgumentIdentifierProperties() * @var array */ private $argumentIdentifierProperties; /** * Initialize the service before usage with the object to be worked with * * @param \WE\SpreadsheetImport\Domain\Model\SpreadsheetImport $spreadsheetImport * * @return $this */ public function init(SpreadsheetImport $spreadsheetImport) { $this->inverseMappingProperties = array(); $this->mappingIdentifierProperties = array(); $this->argumentIdentifierProperties = array(); $this->spreadsheetImport = $spreadsheetImport; $this->domain = $this->settings[$spreadsheetImport->getContext()]['domain']; $context = $spreadsheetImport->getContext(); $this->domain = $this->settings[$context]['domain']; return $this; } /** * Returns the annotation properties of the related domain model with the property name as key. * * Example: * array(['name'] => :Mapping, ['id'] => :Mapping) * * @return array */ public function getAnnotationMappingProperties() { Loading @@ -100,6 +141,8 @@ class SpreadsheetImportService { } /** * Returns the total records of the spreadsheet excluding the header row * * @return array */ public function getTotalRecords() { Loading @@ -110,6 +153,11 @@ class SpreadsheetImportService { } /** * Returns the spreadsheet column as key value pair together with the heading taken of the first row * * Example: * array(['A'] => 'firstname', ['C'] => 'id') * * @return array */ public function getSpreadsheetColumns() { Loading @@ -130,6 +178,8 @@ class SpreadsheetImportService { } /** * Returns a completely mapped new domain object with values taken of the specified row in the spreadsheet. * * @param int $number * * @return object Loading @@ -145,6 +195,9 @@ class SpreadsheetImportService { } /** * Run the import. It inserts, updates and deletes dependent on the configuration of the initialized * SpreadsheetImport object. * * @return void */ public function import() { Loading @@ -153,15 +206,14 @@ class SpreadsheetImportService { $totalDeleted = 0; $processedObjectIds = array(); $objectRepository = $this->getDomainRepository(); $objectValidator = $this->validatorResolver->getBaseValidatorConjunction($this->domain); $identifierProperties = $this->getDomainMappingIdentifierProperties(); $objectValidator = $this->validatorResolver->getBaseValidatorConjunction($this->domain, self::VALIDATION_GROUPS); $sheet = $this->getFileActiveSheet(); $persistRecordsChunkSize = intval($this->settings['persistRecordsChunkSize']); $totalCount = 0; /** @var \PHPExcel_Worksheet_Row $row */ foreach ($sheet->getRowIterator(2) as $row) { $totalCount++; $object = $this->findObjectByIdentifierPropertiesPerRow($identifierProperties, $row); $object = $this->findExistingObjectByRow($row); if (is_object($object)) { $processedObjectIds[] = $this->persistenceManager->getIdentifierByObject($object); if ($this->spreadsheetImport->isUpdating()) { Loading Loading @@ -194,7 +246,7 @@ class SpreadsheetImportService { } $deleteCount = 0; if ($this->spreadsheetImport->isDeleting()) { $notExistingObjects = $this->findObjectsByExcludedIds($processedObjectIds); $notExistingObjects = $this->findObjectsByArgumentsAndExcludedIds($processedObjectIds); foreach ($notExistingObjects as $object) { $objectRepository->remove($object); if (++$deleteCount % $persistRecordsChunkSize === 0) { Loading @@ -210,6 +262,8 @@ class SpreadsheetImportService { } /** * Return the active sheet of a spreadsheet. Other potential sheets are ignored on the import. * * @return \PHPExcel_Worksheet */ private function getFileActiveSheet() { Loading @@ -220,52 +274,30 @@ class SpreadsheetImportService { } /** * @return \TYPO3\Flow\Persistence\RepositoryInterface */ private function getDomainRepository() { $repositoryClassName = preg_replace(array('/\\\Model\\\/', '/$/'), array('\\Repository\\', 'Repository'), $this->domain); /** @var RepositoryInterface $repository */ $repository = $this->objectManager->get($repositoryClassName); return $repository; } /** * @return array */ private function getDomainMappingIdentifierProperties() { // TODO: Don't use the annotation properties but the SpreadsheetImport mapping since we store the Mapping object there as well $domainMappingProperties = array(); $properties = $this->reflectionService->getPropertyNamesByAnnotation($this->domain, Mapping::class); foreach ($properties as $property) { /** @var Mapping $mapping */ $mapping = $this->reflectionService->getPropertyAnnotation($this->domain, $property, Mapping::class); if ($mapping->identifier) { $domainMappingProperties[$property] = $mapping; } } return $domainMappingProperties; } /** * @param array $identifierProperties * Returns an existing object found or null for a specific row. * * @param \PHPExcel_Worksheet_Row $row * * @return null|object */ private function findObjectByIdentifierPropertiesPerRow(array $identifierProperties, \PHPExcel_Worksheet_Row $row) { private function findExistingObjectByRow(\PHPExcel_Worksheet_Row $row) { $query = $this->getDomainRepository()->createQuery(); $constraints = array(); $spreadsheetImportMapping = $this->spreadsheetImport->getMapping(); $mappingIdentifierProperties = $this->getMappingIdentifierProperties(); foreach ($mappingIdentifierProperties as $property => $columnMapping) { $column = $columnMapping['column']; /** @var Mapping $mapping */ foreach ($identifierProperties as $property => $mapping) { $column = $spreadsheetImportMapping[$property]['column']; $mapping = $columnMapping['mapping']; $propertyName = $mapping->queryPropertyName ?: $property; /** @var \PHPExcel_Worksheet_RowCellIterator $cellIterator */ $cellIterator = $row->getCellIterator($column, $column); $value = $cellIterator->current()->getValue(); $propertyName = $mapping->queryPropertyName ?: $property; $constraints[] = $query->equals($propertyName, $value); } $this->mergeQueryConstraintsWithArgumentIdentifiers($query, $constraints); $argumentIdentifierProperties = $this->getArgumentIdentifierProperties(); foreach ($argumentIdentifierProperties as $property => $value) { $constraints[] = $query->equals($property, $value); } if (!empty($constraints)) { return $query->matching($query->logicalAnd($constraints))->execute()->getFirst(); } else { Loading @@ -274,68 +306,64 @@ class SpreadsheetImportService { } /** * @param \TYPO3\Flow\Persistence\QueryInterface $query * @param array $constraints * @param array $identifiers * * @return \TYPO3\Flow\Persistence\QueryResultInterface */ private function mergeQueryConstraintsWithArgumentIdentifiers(QueryInterface $query, &$constraints) { $contextArguments = $this->settings[$this->spreadsheetImport->getContext()]['arguments']; if (is_array($contextArguments)) { foreach ($contextArguments as $contextArgument) { if (isset($contextArgument['identifier']) && $contextArgument['identifier'] == TRUE) { $name = $contextArgument['name']; $arguments = $this->spreadsheetImport->getArguments(); if (array_key_exists($name, $arguments)) { $value = $arguments[$name]; private function findObjectsByArgumentsAndExcludedIds(array $identifiers) { $query = $this->getDomainRepository()->createQuery(); $constraints[] = $query->logicalNot($query->in('Persistence_Object_Identifier', $identifiers)); // Include all the arguments into the query to get a subset only foreach ($this->spreadsheetImport->getArguments() as $name => $value) { $constraints[] = $query->equals($name, $value); } return $query->matching($query->logicalAnd($constraints))->execute(); } } } /** * @param object $object * @param \PHPExcel_Worksheet_Row $row */ private function setObjectPropertiesByRow($object, $row) { /* Set the argument properties before the mapping properties, as mapping property setters might be dependent on argument property values */ $this->setObjectArgumentProperties($object); $this->setObjectMappingProperties($object, $row); } /** * @param \TYPO3\Flow\Persistence\QueryInterface $query * @param array $constraints * @param $object */ private function mergeQueryConstraintsWithArguments(QueryInterface $query, &$constraints) { private function setObjectArgumentProperties($object) { $contextArguments = $this->settings[$this->spreadsheetImport->getContext()]['arguments']; if (is_array($contextArguments)) { $arguments = $this->spreadsheetImport->getArguments(); foreach ($contextArguments as $contextArgument) { $name = $contextArgument['name']; $arguments = $this->spreadsheetImport->getArguments(); if (array_key_exists($name, $arguments)) { if (array_key_exists('domain', $contextArgument)) { $value = $this->propertyMapper->convert($arguments[$name], $contextArgument['domain']); } else { $value = $arguments[$name]; $constraints[] = $query->equals($name, $value); } $setter = 'set' . ucfirst($name); $object->$setter($value); } } } /** * @param array $identifiers * * @return \TYPO3\Flow\Persistence\QueryResultInterface */ private function findObjectsByExcludedIds(array $identifiers) { $query = $this->getDomainRepository()->createQuery(); $constraints[] = $query->logicalNot($query->in('Persistence_Object_Identifier', $identifiers)); $this->mergeQueryConstraintsWithArguments($query, $constraints); return $query->matching($query->logicalAnd($constraints))->execute(); } /** * @param object $object * @param \PHPExcel_Worksheet_Row $row */ private function setObjectPropertiesByRow($object, $row) { // Set the arguments first as mapping property setters might be dependent on argument properties $this->setObjectArgumentProperties($object); $inverseSpreadsheetImportMapping = $this->getInverseSpreadsheetImportMapping(); private function setObjectMappingProperties($object, $row) { $inversedMappingProperties = $this->getInverseMappingProperties(); /** @var \PHPExcel_Cell $cell */ foreach ($row->getCellIterator() as $cell) { $column = $cell->getColumn(); if (array_key_exists($column, $inverseSpreadsheetImportMapping)) { $properties = $inverseSpreadsheetImportMapping[$column]; if (array_key_exists($column, $inversedMappingProperties)) { $properties = $inversedMappingProperties[$column]; foreach ($properties as $propertyMapping) { $property = $propertyMapping['property']; /** @var Mapping $mapping */ Loading @@ -349,40 +377,84 @@ class SpreadsheetImportService { /** * Return an inverse SpreadsheetImport mapping array. It flips the property and column attribute and returns it as a * 3-dim array instead of a 2-dim array. The reason for that is the case when the same column is assigned to multiple * 3-dim array instead of a 2-dim array. This is done for the case when the same column is assigned to multiple * properties. * * Example: * array( * ['A'] => array( * [0] => array(['property'] => 'name', ['mapping'] => :Mapping), * [1] => array(['property'] => 'id', ['mapping'] => :Mapping))); * * @return array */ private function getInverseSpreadsheetImportMapping() { if (empty($this->inverseSpreadsheetImportMapping)) { $this->inverseSpreadsheetImportMapping = array(); private function getInverseMappingProperties() { if (empty($this->inverseMappingProperties)) { $this->inverseMappingProperties = array(); foreach ($this->spreadsheetImport->getMapping() as $property => $columnMapping) { $column = $columnMapping['column']; $propertyMapping = array('property' => $property, 'mapping' => $columnMapping['mapping']); $this->inverseSpreadsheetImportMapping[$column][] = $propertyMapping; $this->inverseMappingProperties[$column][] = $propertyMapping; } } return $this->inverseSpreadsheetImportMapping; return $this->inverseMappingProperties; } /** * @param $object * Filters the SpreadsheetImport mappings elements for Mapping identifiers only. * * Example: * array(['id'] => array(['column'] => 'A', ['mapping'] => :Mapping))); * * @return array */ private function setObjectArgumentProperties($object) { $contextArguments = $this->settings[$this->spreadsheetImport->getContext()]['arguments']; private function getMappingIdentifierProperties() { if (empty($this->mappingIdentifierProperties)) { foreach ($this->spreadsheetImport->getMapping() as $property => $columnMapping) { /** @var Mapping $mapping */ $mapping = $columnMapping['mapping']; if ($mapping->identifier) { $this->mappingIdentifierProperties[$property] = $columnMapping; } } } return $this->mappingIdentifierProperties; } /** * Filters the SpreadsheetImport argument elements for identifier arguments only. * * Example: * array(['id'] => '8df83181-6fb2-11e4-8b37-00505601050c'); * * @return array */ private function getArgumentIdentifierProperties() { if (empty($this->argumentIdentifierProperties)) { $context = $this->spreadsheetImport->getContext(); $contextArguments = $this->settings[$context]['arguments']; if (is_array($contextArguments)) { $arguments = $this->spreadsheetImport->getArguments(); foreach ($contextArguments as $contextArgument) { if (isset($contextArgument['identifier']) && $contextArgument['identifier'] == TRUE) { $name = $contextArgument['name']; $arguments = $this->spreadsheetImport->getArguments(); if (array_key_exists($name, $arguments)) { if (array_key_exists('domain', $contextArgument)) { $value = $this->propertyMapper->convert($arguments[$name], $contextArgument['domain']); } else { $value = $arguments[$name]; $this->argumentIdentifierProperties[$name] = $arguments[$name]; } } $setter = 'set' . ucfirst($name); $object->$setter($value); } } } return $this->argumentIdentifierProperties; } /** * @return \TYPO3\Flow\Persistence\RepositoryInterface */ private function getDomainRepository() { $repositoryClassName = preg_replace(array('/\\\Model\\\/', '/$/'), array('\\Repository\\', 'Repository'), $this->domain); /** @var RepositoryInterface $repository */ $repository = $this->objectManager->get($repositoryClassName); return $repository; } } Loading
Classes/WE/SpreadsheetImport/Command/SpreadsheetImportCommandController.php +49 −19 Original line number Diff line number Diff line Loading @@ -51,7 +51,7 @@ class SpreadsheetImportCommandController extends CommandController { public function importCommand() { $currentImportingCount = $this->spreadsheetImportRepository->countByImportingStatus(SpreadsheetImport::IMPORTING_STATUS_IN_PROGRESS); if ($currentImportingCount > 0) { $this->outputFormatted('Previous spreadsheet import is still in progress.'); $this->outputLine('Previous spreadsheet import is still in progress.'); $this->quit(); } /** @var SpreadsheetImport $spreadsheetImport */ Loading @@ -67,35 +67,65 @@ class SpreadsheetImportCommandController extends CommandController { try { $this->spreadsheetImportService->import(); $spreadsheetImport->setImportingStatus(SpreadsheetImport::IMPORTING_STATUS_COMPLETED); $this->outputFormatted('Spreadsheet has been imported. %d inserted, %d updated, %d deleted, %d skipped', $this->outputLine('Spreadsheet has been imported. %d inserted, %d updated, %d deleted, %d skipped', array($spreadsheetImport->getTotalInserted(), $spreadsheetImport->getTotalUpdated(), $spreadsheetImport->getTotalDeleted(), $spreadsheetImport->getTotalSkipped())); } catch (Exception $e) { } catch (\Exception $e) { $spreadsheetImport->setImportingStatus(SpreadsheetImport::IMPORTING_STATUS_FAILED); $this->outputFormatted('Spreadsheet import failed.'); $this->outputLine('Spreadsheet import failed.'); } try { $this->spreadsheetImportRepository->update($spreadsheetImport); } catch (\Exception $e) { $this->outputLine('Spreadsheet import status update error. It remains in progress until cleanup.'); } } else { $this->outputFormatted('No spreadsheet import in queue.'); } } /** * Cleanup previous spreadsheet imports. Threashold defined in settings. * Cleanup past and stalled imports. * * @param int $keepPastImportsThreasholdDays Overwrites the setting value * @param int $maxExecutionThreasholdMinutes Overwrites the setting value */ public function cleanupCommand() { $cleanupImportsThreasholdDays = intval($this->settings['cleanupImportsThreasholdDays']); $cleanupFromDate = new \DateTime(); $cleanupFromDate->sub(new \DateInterval('P' . $cleanupImportsThreasholdDays . 'D')); $oldSpreadsheetImports = $this->spreadsheetImportRepository->findPreviousImportsBySpecificDate($cleanupFromDate); if ($oldSpreadsheetImports->count() > 0) { /** @var SpreadsheetImport $oldSpreadsheetImport */ foreach ($oldSpreadsheetImports as $oldSpreadsheetImport) { $this->spreadsheetImportRepository->remove($oldSpreadsheetImport); public function cleanupCommand($keepPastImportsThreasholdDays = -1, $maxExecutionThreasholdMinutes = -1) { $keepPastImportsThreasholdDays = ($keepPastImportsThreasholdDays >= 0) ? $keepPastImportsThreasholdDays : intval($this->settings['keepPastImportsThreasholdDays']); $maxExecutionThreasholdMinutes = ($maxExecutionThreasholdMinutes >= 0) ? $maxExecutionThreasholdMinutes : intval($this->settings['maxExecutionThreasholdMinutes']); $this->cleanupPastImports($keepPastImportsThreasholdDays); $this->cleanupStalledImports($maxExecutionThreasholdMinutes); } $this->outputLine('%d spreadsheet imports were removed.', array($oldSpreadsheetImports->count())); } else { $this->outputLine('There is no spreadsheet import in queue to remove.'); /** * Delete past imports * * @param int $keepPastImportsThreasholdDays */ private function cleanupPastImports($keepPastImportsThreasholdDays) { $cleanupFromDateTime = new \DateTime(); $cleanupFromDateTime->sub(new \DateInterval('P' . $keepPastImportsThreasholdDays . 'D')); $spreadsheetImports = $this->spreadsheetImportRepository->findBySpecificDateTimeAndImportingStatus($cleanupFromDateTime); /** @var SpreadsheetImport $spreadsheetImport */ foreach ($spreadsheetImports as $spreadsheetImport) { $this->spreadsheetImportRepository->remove($spreadsheetImport); } $this->outputLine('%d spreadsheet imports removed.', array($spreadsheetImports->count())); } /** * Set stalled imports to failed * * @param int $maxExecutionThreasholdMinutes */ private function cleanupStalledImports($maxExecutionThreasholdMinutes) { $cleanupFromDateTime = new \DateTime(); $cleanupFromDateTime->sub(new \DateInterval('PT' . $maxExecutionThreasholdMinutes . 'M')); $spreadsheetImports = $this->spreadsheetImportRepository->findBySpecificDateTimeAndImportingStatus($cleanupFromDateTime, SpreadsheetImport::IMPORTING_STATUS_IN_PROGRESS); /** @var SpreadsheetImport $spreadsheetImport */ foreach ($spreadsheetImports as $spreadsheetImport) { $spreadsheetImport->setImportingStatus(SpreadsheetImport::IMPORTING_STATUS_FAILED); $this->spreadsheetImportRepository->update($spreadsheetImport); } $this->outputLine('%d spreadsheet imports set to failed.', array($spreadsheetImports->count())); } }
Classes/WE/SpreadsheetImport/Domain/Model/SpreadsheetImport.php +19 −1 Original line number Diff line number Diff line Loading @@ -38,10 +38,19 @@ class SpreadsheetImport { protected $file; /** * DateTime when import is scheduled to progress * * @var \DateTime */ protected $scheduleDate; /** * Actual DateTime when import is set to in progress * * @var \DateTime */ protected $progressDate; /** * @var string * @ORM\Column(type="text") Loading Loading @@ -100,6 +109,7 @@ class SpreadsheetImport { */ public function __construct() { $this->scheduleDate = new \DateTime(); $this->progressDate = new \DateTime(); } /** Loading Loading @@ -142,6 +152,7 @@ class SpreadsheetImport { */ public function setScheduleDate($scheduleDate) { $this->scheduleDate = $scheduleDate; $this->progressDate = $scheduleDate; } /** Loading @@ -166,7 +177,11 @@ class SpreadsheetImport { * @return array */ public function getArguments() { return unserialize($this->arguments); $arguments = unserialize($this->arguments); if (! is_array($arguments)) { $arguments = array(); } return $arguments; } /** Loading Loading @@ -234,6 +249,9 @@ class SpreadsheetImport { */ public function setImportingStatus($importingStatus) { $this->importingStatus = $importingStatus; if ($importingStatus === self::IMPORTING_STATUS_IN_PROGRESS) { $this->progressDate = new \DateTime(); } } /** Loading
Classes/WE/SpreadsheetImport/Domain/Repository/SpreadsheetImportRepository.php +6 −2 Original line number Diff line number Diff line Loading @@ -42,12 +42,16 @@ class SpreadsheetImportRepository extends Repository { /** * @param \DateTime $dateTime * @param int $importingStatus * * @return \TYPO3\Flow\Persistence\QueryResultInterface */ public function findPreviousImportsBySpecificDate(\DateTime $dateTime) { public function findBySpecificDateTimeAndImportingStatus(\DateTime $dateTime, $importingStatus = -1) { $query = $this->createQuery(); $constraint = $query->lessThanOrEqual('scheduleDate', $dateTime); $constraint = $query->lessThanOrEqual('progressDate', $dateTime); if ($importingStatus >= 0) { $constraint = $query->logicalAnd($constraint, $query->equals('importingStatus', $importingStatus)); } return $query->matching($constraint)->execute(); } Loading
Classes/WE/SpreadsheetImport/FrontendMappingService.php +1 −1 Original line number Diff line number Diff line Loading @@ -122,7 +122,7 @@ class FrontendMappingService { $previewObject = $this->spreadsheetImportService->getObjectByRow($record); $preview = array(); $hasErrors = FALSE; $objectValidator = $this->validatorResolver->getBaseValidatorConjunction($domain); $objectValidator = $this->validatorResolver->getBaseValidatorConjunction($domain, SpreadsheetImportService::VALIDATION_GROUPS); $errors = $objectValidator->validate($previewObject)->getFlattenedErrors(); foreach ($mapping as $property => $columnMapping) { /** @var Mapping $mapping */ Loading
Classes/WE/SpreadsheetImport/SpreadsheetImportService.php +174 −102 Original line number Diff line number Diff line Loading @@ -12,22 +12,29 @@ namespace WE\SpreadsheetImport; * */ use TYPO3\Flow\Annotations as Flow; use TYPO3\Flow\Persistence\QueryInterface; use TYPO3\Flow\Persistence\RepositoryInterface; use WE\SpreadsheetImport\Annotations\Mapping; use WE\SpreadsheetImport\Domain\Model\SpreadsheetImport; /** * Service to handle the column mapping and the import itself * * @Flow\Scope("singleton") */ class SpreadsheetImportService { const VALIDATION_GROUPS = array('Default', 'SpreadsheetImport'); /** * Domain object containing the configuration the service works with * * @var SpreadsheetImport */ protected $spreadsheetImport; /** * Class name of the domain model to be imported * * @var string */ protected $domain; Loading Loading @@ -69,25 +76,59 @@ class SpreadsheetImportService { protected $validatorResolver; /** * Inverse SpreadsheetImport mapping array * Inverse array of SpreadsheetDomain mapping array property. Always use the getter function, which will process the * property in case it is not set. * * @internal * @see SpreadsheetImportService::getInverseMappingProperties() * @var array */ private $inverseMappingProperties; /** * Identifier properties of SpreadsheetDomain mapping array. Always use the getter function, which will process the * property in case it is not set. * * @internal * @see SpreadsheetImportService::getMappingIdentifierProperties() * @var array */ private $inverseSpreadsheetImportMapping; private $mappingIdentifierProperties; /** * Identifier properties of SpreadsheetDomain argument array. Always use the getter function, which will process the * property in case it is not set. * * @internal * @see SpreadsheetImportService::getArgumentIdentifierProperties() * @var array */ private $argumentIdentifierProperties; /** * Initialize the service before usage with the object to be worked with * * @param \WE\SpreadsheetImport\Domain\Model\SpreadsheetImport $spreadsheetImport * * @return $this */ public function init(SpreadsheetImport $spreadsheetImport) { $this->inverseMappingProperties = array(); $this->mappingIdentifierProperties = array(); $this->argumentIdentifierProperties = array(); $this->spreadsheetImport = $spreadsheetImport; $this->domain = $this->settings[$spreadsheetImport->getContext()]['domain']; $context = $spreadsheetImport->getContext(); $this->domain = $this->settings[$context]['domain']; return $this; } /** * Returns the annotation properties of the related domain model with the property name as key. * * Example: * array(['name'] => :Mapping, ['id'] => :Mapping) * * @return array */ public function getAnnotationMappingProperties() { Loading @@ -100,6 +141,8 @@ class SpreadsheetImportService { } /** * Returns the total records of the spreadsheet excluding the header row * * @return array */ public function getTotalRecords() { Loading @@ -110,6 +153,11 @@ class SpreadsheetImportService { } /** * Returns the spreadsheet column as key value pair together with the heading taken of the first row * * Example: * array(['A'] => 'firstname', ['C'] => 'id') * * @return array */ public function getSpreadsheetColumns() { Loading @@ -130,6 +178,8 @@ class SpreadsheetImportService { } /** * Returns a completely mapped new domain object with values taken of the specified row in the spreadsheet. * * @param int $number * * @return object Loading @@ -145,6 +195,9 @@ class SpreadsheetImportService { } /** * Run the import. It inserts, updates and deletes dependent on the configuration of the initialized * SpreadsheetImport object. * * @return void */ public function import() { Loading @@ -153,15 +206,14 @@ class SpreadsheetImportService { $totalDeleted = 0; $processedObjectIds = array(); $objectRepository = $this->getDomainRepository(); $objectValidator = $this->validatorResolver->getBaseValidatorConjunction($this->domain); $identifierProperties = $this->getDomainMappingIdentifierProperties(); $objectValidator = $this->validatorResolver->getBaseValidatorConjunction($this->domain, self::VALIDATION_GROUPS); $sheet = $this->getFileActiveSheet(); $persistRecordsChunkSize = intval($this->settings['persistRecordsChunkSize']); $totalCount = 0; /** @var \PHPExcel_Worksheet_Row $row */ foreach ($sheet->getRowIterator(2) as $row) { $totalCount++; $object = $this->findObjectByIdentifierPropertiesPerRow($identifierProperties, $row); $object = $this->findExistingObjectByRow($row); if (is_object($object)) { $processedObjectIds[] = $this->persistenceManager->getIdentifierByObject($object); if ($this->spreadsheetImport->isUpdating()) { Loading Loading @@ -194,7 +246,7 @@ class SpreadsheetImportService { } $deleteCount = 0; if ($this->spreadsheetImport->isDeleting()) { $notExistingObjects = $this->findObjectsByExcludedIds($processedObjectIds); $notExistingObjects = $this->findObjectsByArgumentsAndExcludedIds($processedObjectIds); foreach ($notExistingObjects as $object) { $objectRepository->remove($object); if (++$deleteCount % $persistRecordsChunkSize === 0) { Loading @@ -210,6 +262,8 @@ class SpreadsheetImportService { } /** * Return the active sheet of a spreadsheet. Other potential sheets are ignored on the import. * * @return \PHPExcel_Worksheet */ private function getFileActiveSheet() { Loading @@ -220,52 +274,30 @@ class SpreadsheetImportService { } /** * @return \TYPO3\Flow\Persistence\RepositoryInterface */ private function getDomainRepository() { $repositoryClassName = preg_replace(array('/\\\Model\\\/', '/$/'), array('\\Repository\\', 'Repository'), $this->domain); /** @var RepositoryInterface $repository */ $repository = $this->objectManager->get($repositoryClassName); return $repository; } /** * @return array */ private function getDomainMappingIdentifierProperties() { // TODO: Don't use the annotation properties but the SpreadsheetImport mapping since we store the Mapping object there as well $domainMappingProperties = array(); $properties = $this->reflectionService->getPropertyNamesByAnnotation($this->domain, Mapping::class); foreach ($properties as $property) { /** @var Mapping $mapping */ $mapping = $this->reflectionService->getPropertyAnnotation($this->domain, $property, Mapping::class); if ($mapping->identifier) { $domainMappingProperties[$property] = $mapping; } } return $domainMappingProperties; } /** * @param array $identifierProperties * Returns an existing object found or null for a specific row. * * @param \PHPExcel_Worksheet_Row $row * * @return null|object */ private function findObjectByIdentifierPropertiesPerRow(array $identifierProperties, \PHPExcel_Worksheet_Row $row) { private function findExistingObjectByRow(\PHPExcel_Worksheet_Row $row) { $query = $this->getDomainRepository()->createQuery(); $constraints = array(); $spreadsheetImportMapping = $this->spreadsheetImport->getMapping(); $mappingIdentifierProperties = $this->getMappingIdentifierProperties(); foreach ($mappingIdentifierProperties as $property => $columnMapping) { $column = $columnMapping['column']; /** @var Mapping $mapping */ foreach ($identifierProperties as $property => $mapping) { $column = $spreadsheetImportMapping[$property]['column']; $mapping = $columnMapping['mapping']; $propertyName = $mapping->queryPropertyName ?: $property; /** @var \PHPExcel_Worksheet_RowCellIterator $cellIterator */ $cellIterator = $row->getCellIterator($column, $column); $value = $cellIterator->current()->getValue(); $propertyName = $mapping->queryPropertyName ?: $property; $constraints[] = $query->equals($propertyName, $value); } $this->mergeQueryConstraintsWithArgumentIdentifiers($query, $constraints); $argumentIdentifierProperties = $this->getArgumentIdentifierProperties(); foreach ($argumentIdentifierProperties as $property => $value) { $constraints[] = $query->equals($property, $value); } if (!empty($constraints)) { return $query->matching($query->logicalAnd($constraints))->execute()->getFirst(); } else { Loading @@ -274,68 +306,64 @@ class SpreadsheetImportService { } /** * @param \TYPO3\Flow\Persistence\QueryInterface $query * @param array $constraints * @param array $identifiers * * @return \TYPO3\Flow\Persistence\QueryResultInterface */ private function mergeQueryConstraintsWithArgumentIdentifiers(QueryInterface $query, &$constraints) { $contextArguments = $this->settings[$this->spreadsheetImport->getContext()]['arguments']; if (is_array($contextArguments)) { foreach ($contextArguments as $contextArgument) { if (isset($contextArgument['identifier']) && $contextArgument['identifier'] == TRUE) { $name = $contextArgument['name']; $arguments = $this->spreadsheetImport->getArguments(); if (array_key_exists($name, $arguments)) { $value = $arguments[$name]; private function findObjectsByArgumentsAndExcludedIds(array $identifiers) { $query = $this->getDomainRepository()->createQuery(); $constraints[] = $query->logicalNot($query->in('Persistence_Object_Identifier', $identifiers)); // Include all the arguments into the query to get a subset only foreach ($this->spreadsheetImport->getArguments() as $name => $value) { $constraints[] = $query->equals($name, $value); } return $query->matching($query->logicalAnd($constraints))->execute(); } } } /** * @param object $object * @param \PHPExcel_Worksheet_Row $row */ private function setObjectPropertiesByRow($object, $row) { /* Set the argument properties before the mapping properties, as mapping property setters might be dependent on argument property values */ $this->setObjectArgumentProperties($object); $this->setObjectMappingProperties($object, $row); } /** * @param \TYPO3\Flow\Persistence\QueryInterface $query * @param array $constraints * @param $object */ private function mergeQueryConstraintsWithArguments(QueryInterface $query, &$constraints) { private function setObjectArgumentProperties($object) { $contextArguments = $this->settings[$this->spreadsheetImport->getContext()]['arguments']; if (is_array($contextArguments)) { $arguments = $this->spreadsheetImport->getArguments(); foreach ($contextArguments as $contextArgument) { $name = $contextArgument['name']; $arguments = $this->spreadsheetImport->getArguments(); if (array_key_exists($name, $arguments)) { if (array_key_exists('domain', $contextArgument)) { $value = $this->propertyMapper->convert($arguments[$name], $contextArgument['domain']); } else { $value = $arguments[$name]; $constraints[] = $query->equals($name, $value); } $setter = 'set' . ucfirst($name); $object->$setter($value); } } } /** * @param array $identifiers * * @return \TYPO3\Flow\Persistence\QueryResultInterface */ private function findObjectsByExcludedIds(array $identifiers) { $query = $this->getDomainRepository()->createQuery(); $constraints[] = $query->logicalNot($query->in('Persistence_Object_Identifier', $identifiers)); $this->mergeQueryConstraintsWithArguments($query, $constraints); return $query->matching($query->logicalAnd($constraints))->execute(); } /** * @param object $object * @param \PHPExcel_Worksheet_Row $row */ private function setObjectPropertiesByRow($object, $row) { // Set the arguments first as mapping property setters might be dependent on argument properties $this->setObjectArgumentProperties($object); $inverseSpreadsheetImportMapping = $this->getInverseSpreadsheetImportMapping(); private function setObjectMappingProperties($object, $row) { $inversedMappingProperties = $this->getInverseMappingProperties(); /** @var \PHPExcel_Cell $cell */ foreach ($row->getCellIterator() as $cell) { $column = $cell->getColumn(); if (array_key_exists($column, $inverseSpreadsheetImportMapping)) { $properties = $inverseSpreadsheetImportMapping[$column]; if (array_key_exists($column, $inversedMappingProperties)) { $properties = $inversedMappingProperties[$column]; foreach ($properties as $propertyMapping) { $property = $propertyMapping['property']; /** @var Mapping $mapping */ Loading @@ -349,40 +377,84 @@ class SpreadsheetImportService { /** * Return an inverse SpreadsheetImport mapping array. It flips the property and column attribute and returns it as a * 3-dim array instead of a 2-dim array. The reason for that is the case when the same column is assigned to multiple * 3-dim array instead of a 2-dim array. This is done for the case when the same column is assigned to multiple * properties. * * Example: * array( * ['A'] => array( * [0] => array(['property'] => 'name', ['mapping'] => :Mapping), * [1] => array(['property'] => 'id', ['mapping'] => :Mapping))); * * @return array */ private function getInverseSpreadsheetImportMapping() { if (empty($this->inverseSpreadsheetImportMapping)) { $this->inverseSpreadsheetImportMapping = array(); private function getInverseMappingProperties() { if (empty($this->inverseMappingProperties)) { $this->inverseMappingProperties = array(); foreach ($this->spreadsheetImport->getMapping() as $property => $columnMapping) { $column = $columnMapping['column']; $propertyMapping = array('property' => $property, 'mapping' => $columnMapping['mapping']); $this->inverseSpreadsheetImportMapping[$column][] = $propertyMapping; $this->inverseMappingProperties[$column][] = $propertyMapping; } } return $this->inverseSpreadsheetImportMapping; return $this->inverseMappingProperties; } /** * @param $object * Filters the SpreadsheetImport mappings elements for Mapping identifiers only. * * Example: * array(['id'] => array(['column'] => 'A', ['mapping'] => :Mapping))); * * @return array */ private function setObjectArgumentProperties($object) { $contextArguments = $this->settings[$this->spreadsheetImport->getContext()]['arguments']; private function getMappingIdentifierProperties() { if (empty($this->mappingIdentifierProperties)) { foreach ($this->spreadsheetImport->getMapping() as $property => $columnMapping) { /** @var Mapping $mapping */ $mapping = $columnMapping['mapping']; if ($mapping->identifier) { $this->mappingIdentifierProperties[$property] = $columnMapping; } } } return $this->mappingIdentifierProperties; } /** * Filters the SpreadsheetImport argument elements for identifier arguments only. * * Example: * array(['id'] => '8df83181-6fb2-11e4-8b37-00505601050c'); * * @return array */ private function getArgumentIdentifierProperties() { if (empty($this->argumentIdentifierProperties)) { $context = $this->spreadsheetImport->getContext(); $contextArguments = $this->settings[$context]['arguments']; if (is_array($contextArguments)) { $arguments = $this->spreadsheetImport->getArguments(); foreach ($contextArguments as $contextArgument) { if (isset($contextArgument['identifier']) && $contextArgument['identifier'] == TRUE) { $name = $contextArgument['name']; $arguments = $this->spreadsheetImport->getArguments(); if (array_key_exists($name, $arguments)) { if (array_key_exists('domain', $contextArgument)) { $value = $this->propertyMapper->convert($arguments[$name], $contextArgument['domain']); } else { $value = $arguments[$name]; $this->argumentIdentifierProperties[$name] = $arguments[$name]; } } $setter = 'set' . ucfirst($name); $object->$setter($value); } } } return $this->argumentIdentifierProperties; } /** * @return \TYPO3\Flow\Persistence\RepositoryInterface */ private function getDomainRepository() { $repositoryClassName = preg_replace(array('/\\\Model\\\/', '/$/'), array('\\Repository\\', 'Repository'), $this->domain); /** @var RepositoryInterface $repository */ $repository = $this->objectManager->get($repositoryClassName); return $repository; } }