Loading Classes/WE/SpreadsheetImport/Command/SpreadsheetImportCommandController.php +47 −17 Changes for Classes/WE/SpreadsheetImport/Command/SpreadsheetImportCommandController.php: 47 added lines, 17 removed lines. Original line number Original line Diff line number Diff line Loading @@ -51,7 +51,7 @@ class SpreadsheetImportCommandController extends CommandController { public function importCommand() { public function importCommand() { $currentImportingCount = $this->spreadsheetImportRepository->countByImportingStatus(SpreadsheetImport::IMPORTING_STATUS_IN_PROGRESS); $currentImportingCount = $this->spreadsheetImportRepository->countByImportingStatus(SpreadsheetImport::IMPORTING_STATUS_IN_PROGRESS); if ($currentImportingCount > 0) { if ($currentImportingCount > 0) { $this->outputFormatted('Previous spreadsheet import is still in progress.'); $this->outputLine('Previous spreadsheet import is still in progress.'); $this->quit(); $this->quit(); } } /** @var SpreadsheetImport $spreadsheetImport */ /** @var SpreadsheetImport $spreadsheetImport */ Loading @@ -67,35 +67,65 @@ class SpreadsheetImportCommandController extends CommandController { try { try { $this->spreadsheetImportService->import(); $this->spreadsheetImportService->import(); $spreadsheetImport->setImportingStatus(SpreadsheetImport::IMPORTING_STATUS_COMPLETED); $spreadsheetImport->setImportingStatus(SpreadsheetImport::IMPORTING_STATUS_COMPLETED); $this->outputFormatted('Spreadsheet has been imported. %d inserted, %d updated, %d deleted, %d skipped', $this->outputLine('Spreadsheet has been imported. %d inserted, %d updated, %d deleted, %d skipped', array($spreadsheetImport->getTotalInserted(), $spreadsheetImport->getTotalUpdated(), $spreadsheetImport->getTotalDeleted(), $spreadsheetImport->getTotalSkipped())); array($spreadsheetImport->getTotalInserted(), $spreadsheetImport->getTotalUpdated(), $spreadsheetImport->getTotalDeleted(), $spreadsheetImport->getTotalSkipped())); } catch (Exception $e) { } catch (\Exception $e) { $spreadsheetImport->setImportingStatus(SpreadsheetImport::IMPORTING_STATUS_FAILED); $spreadsheetImport->setImportingStatus(SpreadsheetImport::IMPORTING_STATUS_FAILED); $this->outputFormatted('Spreadsheet import failed.'); $this->outputLine('Spreadsheet import failed.'); } } try { $this->spreadsheetImportRepository->update($spreadsheetImport); $this->spreadsheetImportRepository->update($spreadsheetImport); } catch (\Exception $e) { $this->outputLine('Spreadsheet import status update error. It remains in progress until cleanup.'); } } else { } else { $this->outputFormatted('No spreadsheet import in queue.'); $this->outputFormatted('No spreadsheet import in queue.'); } } } } /** /** * Cleanup previous spreadsheet imports. Threashold defined in settings. * Cleanup past and stalled imports. * * @param int $keepPastImportsThreasholdDays Overwrites the setting value * @param int $maxExecutionThreasholdMinutes Overwrites the setting value */ */ public function cleanupCommand() { public function cleanupCommand($keepPastImportsThreasholdDays = -1, $maxExecutionThreasholdMinutes = -1) { $cleanupImportsThreasholdDays = intval($this->settings['cleanupImportsThreasholdDays']); $keepPastImportsThreasholdDays = ($keepPastImportsThreasholdDays >= 0) ? $keepPastImportsThreasholdDays : intval($this->settings['keepPastImportsThreasholdDays']); $cleanupFromDate = new \DateTime(); $maxExecutionThreasholdMinutes = ($maxExecutionThreasholdMinutes >= 0) ? $maxExecutionThreasholdMinutes : intval($this->settings['maxExecutionThreasholdMinutes']); $cleanupFromDate->sub(new \DateInterval('P' . $cleanupImportsThreasholdDays . 'D')); $this->cleanupPastImports($keepPastImportsThreasholdDays); $oldSpreadsheetImports = $this->spreadsheetImportRepository->findPreviousImportsBySpecificDate($cleanupFromDate); $this->cleanupStalledImports($maxExecutionThreasholdMinutes); if ($oldSpreadsheetImports->count() > 0) { /** @var SpreadsheetImport $oldSpreadsheetImport */ foreach ($oldSpreadsheetImports as $oldSpreadsheetImport) { $this->spreadsheetImportRepository->remove($oldSpreadsheetImport); } } $this->outputLine('%d spreadsheet imports were removed.', array($oldSpreadsheetImports->count())); } else { /** $this->outputLine('There is no spreadsheet import in queue to remove.'); * Delete past imports * * @param int $keepPastImportsThreasholdDays */ private function cleanupPastImports($keepPastImportsThreasholdDays) { $cleanupFromDateTime = new \DateTime(); $cleanupFromDateTime->sub(new \DateInterval('P' . $keepPastImportsThreasholdDays . 'D')); $spreadsheetImports = $this->spreadsheetImportRepository->findBySpecificDateTimeAndImportingStatus($cleanupFromDateTime); /** @var SpreadsheetImport $spreadsheetImport */ foreach ($spreadsheetImports as $spreadsheetImport) { $this->spreadsheetImportRepository->remove($spreadsheetImport); } } $this->outputLine('%d spreadsheet imports removed.', array($spreadsheetImports->count())); } } /** * Set stalled imports to failed * * @param int $maxExecutionThreasholdMinutes */ private function cleanupStalledImports($maxExecutionThreasholdMinutes) { $cleanupFromDateTime = new \DateTime(); $cleanupFromDateTime->sub(new \DateInterval('PT' . $maxExecutionThreasholdMinutes . 'M')); $spreadsheetImports = $this->spreadsheetImportRepository->findBySpecificDateTimeAndImportingStatus($cleanupFromDateTime, SpreadsheetImport::IMPORTING_STATUS_IN_PROGRESS); /** @var SpreadsheetImport $spreadsheetImport */ foreach ($spreadsheetImports as $spreadsheetImport) { $spreadsheetImport->setImportingStatus(SpreadsheetImport::IMPORTING_STATUS_FAILED); $this->spreadsheetImportRepository->update($spreadsheetImport); } $this->outputLine('%d spreadsheet imports set to failed.', array($spreadsheetImports->count())); } } } Classes/WE/SpreadsheetImport/Domain/Model/SpreadsheetImport.php +19 −1 Changes for Classes/WE/SpreadsheetImport/Domain/Model/SpreadsheetImport.php: 19 added lines, 1 removed line. Original line number Original line Diff line number Diff line Loading @@ -38,10 +38,19 @@ class SpreadsheetImport { protected $file; protected $file; /** /** * DateTime when import is scheduled to progress * * @var \DateTime * @var \DateTime */ */ protected $scheduleDate; protected $scheduleDate; /** * Actual DateTime when import is set to in progress * * @var \DateTime */ protected $progressDate; /** /** * @var string * @var string * @ORM\Column(type="text") * @ORM\Column(type="text") Loading Loading @@ -100,6 +109,7 @@ class SpreadsheetImport { */ */ public function __construct() { public function __construct() { $this->scheduleDate = new \DateTime(); $this->scheduleDate = new \DateTime(); $this->progressDate = new \DateTime(); } } /** /** Loading Loading @@ -142,6 +152,7 @@ class SpreadsheetImport { */ */ public function setScheduleDate($scheduleDate) { public function setScheduleDate($scheduleDate) { $this->scheduleDate = $scheduleDate; $this->scheduleDate = $scheduleDate; $this->progressDate = $scheduleDate; } } /** /** Loading @@ -166,7 +177,11 @@ class SpreadsheetImport { * @return array * @return array */ */ public function getArguments() { public function getArguments() { return unserialize($this->arguments); $arguments = unserialize($this->arguments); if (! is_array($arguments)) { $arguments = array(); } return $arguments; } } /** /** Loading Loading @@ -234,6 +249,9 @@ class SpreadsheetImport { */ */ public function setImportingStatus($importingStatus) { public function setImportingStatus($importingStatus) { $this->importingStatus = $importingStatus; $this->importingStatus = $importingStatus; if ($importingStatus === self::IMPORTING_STATUS_IN_PROGRESS) { $this->progressDate = new \DateTime(); } } } /** /** Loading Classes/WE/SpreadsheetImport/Domain/Repository/SpreadsheetImportRepository.php +6 −2 Changes for Classes/WE/SpreadsheetImport/Domain/Repository/SpreadsheetImportRepository.php: 6 added lines, 2 removed lines. Original line number Original line Diff line number Diff line Loading @@ -42,12 +42,16 @@ class SpreadsheetImportRepository extends Repository { /** /** * @param \DateTime $dateTime * @param \DateTime $dateTime * @param int $importingStatus * * * @return \TYPO3\Flow\Persistence\QueryResultInterface * @return \TYPO3\Flow\Persistence\QueryResultInterface */ */ public function findPreviousImportsBySpecificDate(\DateTime $dateTime) { public function findBySpecificDateTimeAndImportingStatus(\DateTime $dateTime, $importingStatus = -1) { $query = $this->createQuery(); $query = $this->createQuery(); $constraint = $query->lessThanOrEqual('scheduleDate', $dateTime); $constraint = $query->lessThanOrEqual('progressDate', $dateTime); if ($importingStatus >= 0) { $constraint = $query->logicalAnd($constraint, $query->equals('importingStatus', $importingStatus)); } return $query->matching($constraint)->execute(); return $query->matching($constraint)->execute(); } } Loading Classes/WE/SpreadsheetImport/FrontendMappingService.php +1 −1 Changes for Classes/WE/SpreadsheetImport/FrontendMappingService.php: 1 added line, 1 removed line. Original line number Original line Diff line number Diff line Loading @@ -122,7 +122,7 @@ class FrontendMappingService { $previewObject = $this->spreadsheetImportService->getObjectByRow($record); $previewObject = $this->spreadsheetImportService->getObjectByRow($record); $preview = array(); $preview = array(); $hasErrors = FALSE; $hasErrors = FALSE; $objectValidator = $this->validatorResolver->getBaseValidatorConjunction($domain); $objectValidator = $this->validatorResolver->getBaseValidatorConjunction($domain, SpreadsheetImportService::VALIDATION_GROUPS); $errors = $objectValidator->validate($previewObject)->getFlattenedErrors(); $errors = $objectValidator->validate($previewObject)->getFlattenedErrors(); foreach ($mapping as $property => $columnMapping) { foreach ($mapping as $property => $columnMapping) { /** @var Mapping $mapping */ /** @var Mapping $mapping */ Loading Classes/WE/SpreadsheetImport/SpreadsheetImportService.php +165 −93 Changes for Classes/WE/SpreadsheetImport/SpreadsheetImportService.php: 165 added lines, 93 removed lines. Original line number Original line Diff line number Diff line Loading @@ -12,22 +12,29 @@ namespace WE\SpreadsheetImport; * */ * */ use TYPO3\Flow\Annotations as Flow; use TYPO3\Flow\Annotations as Flow; use TYPO3\Flow\Persistence\QueryInterface; use TYPO3\Flow\Persistence\RepositoryInterface; use TYPO3\Flow\Persistence\RepositoryInterface; use WE\SpreadsheetImport\Annotations\Mapping; use WE\SpreadsheetImport\Annotations\Mapping; use WE\SpreadsheetImport\Domain\Model\SpreadsheetImport; use WE\SpreadsheetImport\Domain\Model\SpreadsheetImport; /** /** * Service to handle the column mapping and the import itself * * @Flow\Scope("singleton") * @Flow\Scope("singleton") */ */ class SpreadsheetImportService { class SpreadsheetImportService { const VALIDATION_GROUPS = array('Default', 'SpreadsheetImport'); /** /** * Domain object containing the configuration the service works with * * @var SpreadsheetImport * @var SpreadsheetImport */ */ protected $spreadsheetImport; protected $spreadsheetImport; /** /** * Class name of the domain model to be imported * * @var string * @var string */ */ protected $domain; protected $domain; Loading Loading @@ -69,25 +76,59 @@ class SpreadsheetImportService { protected $validatorResolver; protected $validatorResolver; /** /** * Inverse SpreadsheetImport mapping array * Inverse array of SpreadsheetDomain mapping array property. Always use the getter function, which will process the * property in case it is not set. * * @internal * @see SpreadsheetImportService::getInverseMappingProperties() * @var array */ private $inverseMappingProperties; /** * Identifier properties of SpreadsheetDomain mapping array. Always use the getter function, which will process the * property in case it is not set. * * * @internal * @see SpreadsheetImportService::getMappingIdentifierProperties() * @var array * @var array */ */ private $inverseSpreadsheetImportMapping; private $mappingIdentifierProperties; /** /** * Identifier properties of SpreadsheetDomain argument array. Always use the getter function, which will process the * property in case it is not set. * * @internal * @see SpreadsheetImportService::getArgumentIdentifierProperties() * @var array */ private $argumentIdentifierProperties; /** * Initialize the service before usage with the object to be worked with * * @param \WE\SpreadsheetImport\Domain\Model\SpreadsheetImport $spreadsheetImport * @param \WE\SpreadsheetImport\Domain\Model\SpreadsheetImport $spreadsheetImport * * * @return $this * @return $this */ */ public function init(SpreadsheetImport $spreadsheetImport) { public function init(SpreadsheetImport $spreadsheetImport) { $this->inverseMappingProperties = array(); $this->mappingIdentifierProperties = array(); $this->argumentIdentifierProperties = array(); $this->spreadsheetImport = $spreadsheetImport; $this->spreadsheetImport = $spreadsheetImport; $this->domain = $this->settings[$spreadsheetImport->getContext()]['domain']; $context = $spreadsheetImport->getContext(); $this->domain = $this->settings[$context]['domain']; return $this; return $this; } } /** /** * Returns the annotation properties of the related domain model with the property name as key. * * Example: * array(['name'] => :Mapping, ['id'] => :Mapping) * * @return array * @return array */ */ public function getAnnotationMappingProperties() { public function getAnnotationMappingProperties() { Loading @@ -100,6 +141,8 @@ class SpreadsheetImportService { } } /** /** * Returns the total records of the spreadsheet excluding the header row * * @return array * @return array */ */ public function getTotalRecords() { public function getTotalRecords() { Loading @@ -110,6 +153,11 @@ class SpreadsheetImportService { } } /** /** * Returns the spreadsheet column as key value pair together with the heading taken of the first row * * Example: * array(['A'] => 'firstname', ['C'] => 'id') * * @return array * @return array */ */ public function getSpreadsheetColumns() { public function getSpreadsheetColumns() { Loading @@ -130,6 +178,8 @@ class SpreadsheetImportService { } } /** /** * Returns a completely mapped new domain object with values taken of the specified row in the spreadsheet. * * @param int $number * @param int $number * * * @return object * @return object Loading @@ -145,6 +195,9 @@ class SpreadsheetImportService { } } /** /** * Run the import. It inserts, updates and deletes dependent on the configuration of the initialized * SpreadsheetImport object. * * @return void * @return void */ */ public function import() { public function import() { Loading @@ -153,15 +206,14 @@ class SpreadsheetImportService { $totalDeleted = 0; $totalDeleted = 0; $processedObjectIds = array(); $processedObjectIds = array(); $objectRepository = $this->getDomainRepository(); $objectRepository = $this->getDomainRepository(); $objectValidator = $this->validatorResolver->getBaseValidatorConjunction($this->domain); $objectValidator = $this->validatorResolver->getBaseValidatorConjunction($this->domain, self::VALIDATION_GROUPS); $identifierProperties = $this->getDomainMappingIdentifierProperties(); $sheet = $this->getFileActiveSheet(); $sheet = $this->getFileActiveSheet(); $persistRecordsChunkSize = intval($this->settings['persistRecordsChunkSize']); $persistRecordsChunkSize = intval($this->settings['persistRecordsChunkSize']); $totalCount = 0; $totalCount = 0; /** @var \PHPExcel_Worksheet_Row $row */ /** @var \PHPExcel_Worksheet_Row $row */ foreach ($sheet->getRowIterator(2) as $row) { foreach ($sheet->getRowIterator(2) as $row) { $totalCount++; $totalCount++; $object = $this->findObjectByIdentifierPropertiesPerRow($identifierProperties, $row); $object = $this->findExistingObjectByRow($row); if (is_object($object)) { if (is_object($object)) { $processedObjectIds[] = $this->persistenceManager->getIdentifierByObject($object); $processedObjectIds[] = $this->persistenceManager->getIdentifierByObject($object); if ($this->spreadsheetImport->isUpdating()) { if ($this->spreadsheetImport->isUpdating()) { Loading Loading @@ -194,7 +246,7 @@ class SpreadsheetImportService { } } $deleteCount = 0; $deleteCount = 0; if ($this->spreadsheetImport->isDeleting()) { if ($this->spreadsheetImport->isDeleting()) { $notExistingObjects = $this->findObjectsByExcludedIds($processedObjectIds); $notExistingObjects = $this->findObjectsByArgumentsAndExcludedIds($processedObjectIds); foreach ($notExistingObjects as $object) { foreach ($notExistingObjects as $object) { $objectRepository->remove($object); $objectRepository->remove($object); if (++$deleteCount % $persistRecordsChunkSize === 0) { if (++$deleteCount % $persistRecordsChunkSize === 0) { Loading @@ -210,6 +262,8 @@ class SpreadsheetImportService { } } /** /** * Return the active sheet of a spreadsheet. Other potential sheets are ignored on the import. * * @return \PHPExcel_Worksheet * @return \PHPExcel_Worksheet */ */ private function getFileActiveSheet() { private function getFileActiveSheet() { Loading @@ -220,52 +274,30 @@ class SpreadsheetImportService { } } /** /** * @return \TYPO3\Flow\Persistence\RepositoryInterface * Returns an existing object found or null for a specific row. */ * private function getDomainRepository() { $repositoryClassName = preg_replace(array('/\\\Model\\\/', '/$/'), array('\\Repository\\', 'Repository'), $this->domain); /** @var RepositoryInterface $repository */ $repository = $this->objectManager->get($repositoryClassName); return $repository; } /** * @return array */ private function getDomainMappingIdentifierProperties() { // TODO: Don't use the annotation properties but the SpreadsheetImport mapping since we store the Mapping object there as well $domainMappingProperties = array(); $properties = $this->reflectionService->getPropertyNamesByAnnotation($this->domain, Mapping::class); foreach ($properties as $property) { /** @var Mapping $mapping */ $mapping = $this->reflectionService->getPropertyAnnotation($this->domain, $property, Mapping::class); if ($mapping->identifier) { $domainMappingProperties[$property] = $mapping; } } return $domainMappingProperties; } /** * @param array $identifierProperties * @param \PHPExcel_Worksheet_Row $row * @param \PHPExcel_Worksheet_Row $row * * * @return null|object * @return null|object */ */ private function findObjectByIdentifierPropertiesPerRow(array $identifierProperties, \PHPExcel_Worksheet_Row $row) { private function findExistingObjectByRow(\PHPExcel_Worksheet_Row $row) { $query = $this->getDomainRepository()->createQuery(); $query = $this->getDomainRepository()->createQuery(); $constraints = array(); $constraints = array(); $spreadsheetImportMapping = $this->spreadsheetImport->getMapping(); $mappingIdentifierProperties = $this->getMappingIdentifierProperties(); foreach ($mappingIdentifierProperties as $property => $columnMapping) { $column = $columnMapping['column']; /** @var Mapping $mapping */ /** @var Mapping $mapping */ foreach ($identifierProperties as $property => $mapping) { $mapping = $columnMapping['mapping']; $column = $spreadsheetImportMapping[$property]['column']; $propertyName = $mapping->queryPropertyName ?: $property; /** @var \PHPExcel_Worksheet_RowCellIterator $cellIterator */ /** @var \PHPExcel_Worksheet_RowCellIterator $cellIterator */ $cellIterator = $row->getCellIterator($column, $column); $cellIterator = $row->getCellIterator($column, $column); $value = $cellIterator->current()->getValue(); $value = $cellIterator->current()->getValue(); $propertyName = $mapping->queryPropertyName ?: $property; $constraints[] = $query->equals($propertyName, $value); $constraints[] = $query->equals($propertyName, $value); } } $this->mergeQueryConstraintsWithArgumentIdentifiers($query, $constraints); $argumentIdentifierProperties = $this->getArgumentIdentifierProperties(); foreach ($argumentIdentifierProperties as $property => $value) { $constraints[] = $query->equals($property, $value); } if (!empty($constraints)) { if (!empty($constraints)) { return $query->matching($query->logicalAnd($constraints))->execute()->getFirst(); return $query->matching($query->logicalAnd($constraints))->execute()->getFirst(); } else { } else { Loading @@ -274,68 +306,64 @@ class SpreadsheetImportService { } } /** /** * @param \TYPO3\Flow\Persistence\QueryInterface $query * @param array $identifiers * @param array $constraints * * @return \TYPO3\Flow\Persistence\QueryResultInterface */ */ private function mergeQueryConstraintsWithArgumentIdentifiers(QueryInterface $query, &$constraints) { private function findObjectsByArgumentsAndExcludedIds(array $identifiers) { $contextArguments = $this->settings[$this->spreadsheetImport->getContext()]['arguments']; $query = $this->getDomainRepository()->createQuery(); if (is_array($contextArguments)) { $constraints[] = $query->logicalNot($query->in('Persistence_Object_Identifier', $identifiers)); foreach ($contextArguments as $contextArgument) { // Include all the arguments into the query to get a subset only if (isset($contextArgument['identifier']) && $contextArgument['identifier'] == TRUE) { foreach ($this->spreadsheetImport->getArguments() as $name => $value) { $name = $contextArgument['name']; $arguments = $this->spreadsheetImport->getArguments(); if (array_key_exists($name, $arguments)) { $value = $arguments[$name]; $constraints[] = $query->equals($name, $value); $constraints[] = $query->equals($name, $value); } } return $query->matching($query->logicalAnd($constraints))->execute(); } } } } /** * @param object $object * @param \PHPExcel_Worksheet_Row $row */ private function setObjectPropertiesByRow($object, $row) { /* Set the argument properties before the mapping properties, as mapping property setters might be dependent on argument property values */ $this->setObjectArgumentProperties($object); $this->setObjectMappingProperties($object, $row); } } /** /** * @param \TYPO3\Flow\Persistence\QueryInterface $query * @param $object * @param array $constraints */ */ private function mergeQueryConstraintsWithArguments(QueryInterface $query, &$constraints) { private function setObjectArgumentProperties($object) { $contextArguments = $this->settings[$this->spreadsheetImport->getContext()]['arguments']; $contextArguments = $this->settings[$this->spreadsheetImport->getContext()]['arguments']; if (is_array($contextArguments)) { if (is_array($contextArguments)) { $arguments = $this->spreadsheetImport->getArguments(); foreach ($contextArguments as $contextArgument) { foreach ($contextArguments as $contextArgument) { $name = $contextArgument['name']; $name = $contextArgument['name']; $arguments = $this->spreadsheetImport->getArguments(); if (array_key_exists($name, $arguments)) { if (array_key_exists($name, $arguments)) { if (array_key_exists('domain', $contextArgument)) { $value = $this->propertyMapper->convert($arguments[$name], $contextArgument['domain']); } else { $value = $arguments[$name]; $value = $arguments[$name]; $constraints[] = $query->equals($name, $value); } } $setter = 'set' . ucfirst($name); $object->$setter($value); } } } } } } /** * @param array $identifiers * * @return \TYPO3\Flow\Persistence\QueryResultInterface */ private function findObjectsByExcludedIds(array $identifiers) { $query = $this->getDomainRepository()->createQuery(); $constraints[] = $query->logicalNot($query->in('Persistence_Object_Identifier', $identifiers)); $this->mergeQueryConstraintsWithArguments($query, $constraints); return $query->matching($query->logicalAnd($constraints))->execute(); } } /** /** * @param object $object * @param object $object * @param \PHPExcel_Worksheet_Row $row * @param \PHPExcel_Worksheet_Row $row */ */ private function setObjectPropertiesByRow($object, $row) { private function setObjectMappingProperties($object, $row) { // Set the arguments first as mapping property setters might be dependent on argument properties $inversedMappingProperties = $this->getInverseMappingProperties(); $this->setObjectArgumentProperties($object); $inverseSpreadsheetImportMapping = $this->getInverseSpreadsheetImportMapping(); /** @var \PHPExcel_Cell $cell */ /** @var \PHPExcel_Cell $cell */ foreach ($row->getCellIterator() as $cell) { foreach ($row->getCellIterator() as $cell) { $column = $cell->getColumn(); $column = $cell->getColumn(); if (array_key_exists($column, $inverseSpreadsheetImportMapping)) { if (array_key_exists($column, $inversedMappingProperties)) { $properties = $inverseSpreadsheetImportMapping[$column]; $properties = $inversedMappingProperties[$column]; foreach ($properties as $propertyMapping) { foreach ($properties as $propertyMapping) { $property = $propertyMapping['property']; $property = $propertyMapping['property']; /** @var Mapping $mapping */ /** @var Mapping $mapping */ Loading @@ -349,40 +377,84 @@ class SpreadsheetImportService { /** /** * Return an inverse SpreadsheetImport mapping array. It flips the property and column attribute and returns it as a * Return an inverse SpreadsheetImport mapping array. It flips the property and column attribute and returns it as a * 3-dim array instead of a 2-dim array. The reason for that is the case when the same column is assigned to multiple * 3-dim array instead of a 2-dim array. This is done for the case when the same column is assigned to multiple * properties. * properties. * * Example: * array( * ['A'] => array( * [0] => array(['property'] => 'name', ['mapping'] => :Mapping), * [1] => array(['property'] => 'id', ['mapping'] => :Mapping))); * * @return array */ */ private function getInverseSpreadsheetImportMapping() { private function getInverseMappingProperties() { if (empty($this->inverseSpreadsheetImportMapping)) { if (empty($this->inverseMappingProperties)) { $this->inverseSpreadsheetImportMapping = array(); $this->inverseMappingProperties = array(); foreach ($this->spreadsheetImport->getMapping() as $property => $columnMapping) { foreach ($this->spreadsheetImport->getMapping() as $property => $columnMapping) { $column = $columnMapping['column']; $column = $columnMapping['column']; $propertyMapping = array('property' => $property, 'mapping' => $columnMapping['mapping']); $propertyMapping = array('property' => $property, 'mapping' => $columnMapping['mapping']); $this->inverseSpreadsheetImportMapping[$column][] = $propertyMapping; $this->inverseMappingProperties[$column][] = $propertyMapping; } } } } return $this->inverseSpreadsheetImportMapping; return $this->inverseMappingProperties; } } /** /** * @param $object * Filters the SpreadsheetImport mappings elements for Mapping identifiers only. * * Example: * array(['id'] => array(['column'] => 'A', ['mapping'] => :Mapping))); * * @return array */ */ private function setObjectArgumentProperties($object) { private function getMappingIdentifierProperties() { $contextArguments = $this->settings[$this->spreadsheetImport->getContext()]['arguments']; if (empty($this->mappingIdentifierProperties)) { foreach ($this->spreadsheetImport->getMapping() as $property => $columnMapping) { /** @var Mapping $mapping */ $mapping = $columnMapping['mapping']; if ($mapping->identifier) { $this->mappingIdentifierProperties[$property] = $columnMapping; } } } return $this->mappingIdentifierProperties; } /** * Filters the SpreadsheetImport argument elements for identifier arguments only. * * Example: * array(['id'] => '8df83181-6fb2-11e4-8b37-00505601050c'); * * @return array */ private function getArgumentIdentifierProperties() { if (empty($this->argumentIdentifierProperties)) { $context = $this->spreadsheetImport->getContext(); $contextArguments = $this->settings[$context]['arguments']; if (is_array($contextArguments)) { if (is_array($contextArguments)) { $arguments = $this->spreadsheetImport->getArguments(); foreach ($contextArguments as $contextArgument) { foreach ($contextArguments as $contextArgument) { if (isset($contextArgument['identifier']) && $contextArgument['identifier'] == TRUE) { $name = $contextArgument['name']; $name = $contextArgument['name']; $arguments = $this->spreadsheetImport->getArguments(); if (array_key_exists($name, $arguments)) { if (array_key_exists($name, $arguments)) { if (array_key_exists('domain', $contextArgument)) { $this->argumentIdentifierProperties[$name] = $arguments[$name]; $value = $this->propertyMapper->convert($arguments[$name], $contextArgument['domain']); } } else { $value = $arguments[$name]; } } $setter = 'set' . ucfirst($name); $object->$setter($value); } } } } } } return $this->argumentIdentifierProperties; } /** * @return \TYPO3\Flow\Persistence\RepositoryInterface */ private function getDomainRepository() { $repositoryClassName = preg_replace(array('/\\\Model\\\/', '/$/'), array('\\Repository\\', 'Repository'), $this->domain); /** @var RepositoryInterface $repository */ $repository = $this->objectManager->get($repositoryClassName); return $repository; } } } } Loading
Classes/WE/SpreadsheetImport/Command/SpreadsheetImportCommandController.php +47 −17 Changes for Classes/WE/SpreadsheetImport/Command/SpreadsheetImportCommandController.php: 47 added lines, 17 removed lines. Original line number Original line Diff line number Diff line Loading @@ -51,7 +51,7 @@ class SpreadsheetImportCommandController extends CommandController { public function importCommand() { public function importCommand() { $currentImportingCount = $this->spreadsheetImportRepository->countByImportingStatus(SpreadsheetImport::IMPORTING_STATUS_IN_PROGRESS); $currentImportingCount = $this->spreadsheetImportRepository->countByImportingStatus(SpreadsheetImport::IMPORTING_STATUS_IN_PROGRESS); if ($currentImportingCount > 0) { if ($currentImportingCount > 0) { $this->outputFormatted('Previous spreadsheet import is still in progress.'); $this->outputLine('Previous spreadsheet import is still in progress.'); $this->quit(); $this->quit(); } } /** @var SpreadsheetImport $spreadsheetImport */ /** @var SpreadsheetImport $spreadsheetImport */ Loading @@ -67,35 +67,65 @@ class SpreadsheetImportCommandController extends CommandController { try { try { $this->spreadsheetImportService->import(); $this->spreadsheetImportService->import(); $spreadsheetImport->setImportingStatus(SpreadsheetImport::IMPORTING_STATUS_COMPLETED); $spreadsheetImport->setImportingStatus(SpreadsheetImport::IMPORTING_STATUS_COMPLETED); $this->outputFormatted('Spreadsheet has been imported. %d inserted, %d updated, %d deleted, %d skipped', $this->outputLine('Spreadsheet has been imported. %d inserted, %d updated, %d deleted, %d skipped', array($spreadsheetImport->getTotalInserted(), $spreadsheetImport->getTotalUpdated(), $spreadsheetImport->getTotalDeleted(), $spreadsheetImport->getTotalSkipped())); array($spreadsheetImport->getTotalInserted(), $spreadsheetImport->getTotalUpdated(), $spreadsheetImport->getTotalDeleted(), $spreadsheetImport->getTotalSkipped())); } catch (Exception $e) { } catch (\Exception $e) { $spreadsheetImport->setImportingStatus(SpreadsheetImport::IMPORTING_STATUS_FAILED); $spreadsheetImport->setImportingStatus(SpreadsheetImport::IMPORTING_STATUS_FAILED); $this->outputFormatted('Spreadsheet import failed.'); $this->outputLine('Spreadsheet import failed.'); } } try { $this->spreadsheetImportRepository->update($spreadsheetImport); $this->spreadsheetImportRepository->update($spreadsheetImport); } catch (\Exception $e) { $this->outputLine('Spreadsheet import status update error. It remains in progress until cleanup.'); } } else { } else { $this->outputFormatted('No spreadsheet import in queue.'); $this->outputFormatted('No spreadsheet import in queue.'); } } } } /** /** * Cleanup previous spreadsheet imports. Threashold defined in settings. * Cleanup past and stalled imports. * * @param int $keepPastImportsThreasholdDays Overwrites the setting value * @param int $maxExecutionThreasholdMinutes Overwrites the setting value */ */ public function cleanupCommand() { public function cleanupCommand($keepPastImportsThreasholdDays = -1, $maxExecutionThreasholdMinutes = -1) { $cleanupImportsThreasholdDays = intval($this->settings['cleanupImportsThreasholdDays']); $keepPastImportsThreasholdDays = ($keepPastImportsThreasholdDays >= 0) ? $keepPastImportsThreasholdDays : intval($this->settings['keepPastImportsThreasholdDays']); $cleanupFromDate = new \DateTime(); $maxExecutionThreasholdMinutes = ($maxExecutionThreasholdMinutes >= 0) ? $maxExecutionThreasholdMinutes : intval($this->settings['maxExecutionThreasholdMinutes']); $cleanupFromDate->sub(new \DateInterval('P' . $cleanupImportsThreasholdDays . 'D')); $this->cleanupPastImports($keepPastImportsThreasholdDays); $oldSpreadsheetImports = $this->spreadsheetImportRepository->findPreviousImportsBySpecificDate($cleanupFromDate); $this->cleanupStalledImports($maxExecutionThreasholdMinutes); if ($oldSpreadsheetImports->count() > 0) { /** @var SpreadsheetImport $oldSpreadsheetImport */ foreach ($oldSpreadsheetImports as $oldSpreadsheetImport) { $this->spreadsheetImportRepository->remove($oldSpreadsheetImport); } } $this->outputLine('%d spreadsheet imports were removed.', array($oldSpreadsheetImports->count())); } else { /** $this->outputLine('There is no spreadsheet import in queue to remove.'); * Delete past imports * * @param int $keepPastImportsThreasholdDays */ private function cleanupPastImports($keepPastImportsThreasholdDays) { $cleanupFromDateTime = new \DateTime(); $cleanupFromDateTime->sub(new \DateInterval('P' . $keepPastImportsThreasholdDays . 'D')); $spreadsheetImports = $this->spreadsheetImportRepository->findBySpecificDateTimeAndImportingStatus($cleanupFromDateTime); /** @var SpreadsheetImport $spreadsheetImport */ foreach ($spreadsheetImports as $spreadsheetImport) { $this->spreadsheetImportRepository->remove($spreadsheetImport); } } $this->outputLine('%d spreadsheet imports removed.', array($spreadsheetImports->count())); } } /** * Set stalled imports to failed * * @param int $maxExecutionThreasholdMinutes */ private function cleanupStalledImports($maxExecutionThreasholdMinutes) { $cleanupFromDateTime = new \DateTime(); $cleanupFromDateTime->sub(new \DateInterval('PT' . $maxExecutionThreasholdMinutes . 'M')); $spreadsheetImports = $this->spreadsheetImportRepository->findBySpecificDateTimeAndImportingStatus($cleanupFromDateTime, SpreadsheetImport::IMPORTING_STATUS_IN_PROGRESS); /** @var SpreadsheetImport $spreadsheetImport */ foreach ($spreadsheetImports as $spreadsheetImport) { $spreadsheetImport->setImportingStatus(SpreadsheetImport::IMPORTING_STATUS_FAILED); $this->spreadsheetImportRepository->update($spreadsheetImport); } $this->outputLine('%d spreadsheet imports set to failed.', array($spreadsheetImports->count())); } } }
Classes/WE/SpreadsheetImport/Domain/Model/SpreadsheetImport.php +19 −1 Changes for Classes/WE/SpreadsheetImport/Domain/Model/SpreadsheetImport.php: 19 added lines, 1 removed line. Original line number Original line Diff line number Diff line Loading @@ -38,10 +38,19 @@ class SpreadsheetImport { protected $file; protected $file; /** /** * DateTime when import is scheduled to progress * * @var \DateTime * @var \DateTime */ */ protected $scheduleDate; protected $scheduleDate; /** * Actual DateTime when import is set to in progress * * @var \DateTime */ protected $progressDate; /** /** * @var string * @var string * @ORM\Column(type="text") * @ORM\Column(type="text") Loading Loading @@ -100,6 +109,7 @@ class SpreadsheetImport { */ */ public function __construct() { public function __construct() { $this->scheduleDate = new \DateTime(); $this->scheduleDate = new \DateTime(); $this->progressDate = new \DateTime(); } } /** /** Loading Loading @@ -142,6 +152,7 @@ class SpreadsheetImport { */ */ public function setScheduleDate($scheduleDate) { public function setScheduleDate($scheduleDate) { $this->scheduleDate = $scheduleDate; $this->scheduleDate = $scheduleDate; $this->progressDate = $scheduleDate; } } /** /** Loading @@ -166,7 +177,11 @@ class SpreadsheetImport { * @return array * @return array */ */ public function getArguments() { public function getArguments() { return unserialize($this->arguments); $arguments = unserialize($this->arguments); if (! is_array($arguments)) { $arguments = array(); } return $arguments; } } /** /** Loading Loading @@ -234,6 +249,9 @@ class SpreadsheetImport { */ */ public function setImportingStatus($importingStatus) { public function setImportingStatus($importingStatus) { $this->importingStatus = $importingStatus; $this->importingStatus = $importingStatus; if ($importingStatus === self::IMPORTING_STATUS_IN_PROGRESS) { $this->progressDate = new \DateTime(); } } } /** /** Loading
Classes/WE/SpreadsheetImport/Domain/Repository/SpreadsheetImportRepository.php +6 −2 Changes for Classes/WE/SpreadsheetImport/Domain/Repository/SpreadsheetImportRepository.php: 6 added lines, 2 removed lines. Original line number Original line Diff line number Diff line Loading @@ -42,12 +42,16 @@ class SpreadsheetImportRepository extends Repository { /** /** * @param \DateTime $dateTime * @param \DateTime $dateTime * @param int $importingStatus * * * @return \TYPO3\Flow\Persistence\QueryResultInterface * @return \TYPO3\Flow\Persistence\QueryResultInterface */ */ public function findPreviousImportsBySpecificDate(\DateTime $dateTime) { public function findBySpecificDateTimeAndImportingStatus(\DateTime $dateTime, $importingStatus = -1) { $query = $this->createQuery(); $query = $this->createQuery(); $constraint = $query->lessThanOrEqual('scheduleDate', $dateTime); $constraint = $query->lessThanOrEqual('progressDate', $dateTime); if ($importingStatus >= 0) { $constraint = $query->logicalAnd($constraint, $query->equals('importingStatus', $importingStatus)); } return $query->matching($constraint)->execute(); return $query->matching($constraint)->execute(); } } Loading
Classes/WE/SpreadsheetImport/FrontendMappingService.php +1 −1 Changes for Classes/WE/SpreadsheetImport/FrontendMappingService.php: 1 added line, 1 removed line. Original line number Original line Diff line number Diff line Loading @@ -122,7 +122,7 @@ class FrontendMappingService { $previewObject = $this->spreadsheetImportService->getObjectByRow($record); $previewObject = $this->spreadsheetImportService->getObjectByRow($record); $preview = array(); $preview = array(); $hasErrors = FALSE; $hasErrors = FALSE; $objectValidator = $this->validatorResolver->getBaseValidatorConjunction($domain); $objectValidator = $this->validatorResolver->getBaseValidatorConjunction($domain, SpreadsheetImportService::VALIDATION_GROUPS); $errors = $objectValidator->validate($previewObject)->getFlattenedErrors(); $errors = $objectValidator->validate($previewObject)->getFlattenedErrors(); foreach ($mapping as $property => $columnMapping) { foreach ($mapping as $property => $columnMapping) { /** @var Mapping $mapping */ /** @var Mapping $mapping */ Loading
Classes/WE/SpreadsheetImport/SpreadsheetImportService.php +165 −93 Changes for Classes/WE/SpreadsheetImport/SpreadsheetImportService.php: 165 added lines, 93 removed lines. Original line number Original line Diff line number Diff line Loading @@ -12,22 +12,29 @@ namespace WE\SpreadsheetImport; * */ * */ use TYPO3\Flow\Annotations as Flow; use TYPO3\Flow\Annotations as Flow; use TYPO3\Flow\Persistence\QueryInterface; use TYPO3\Flow\Persistence\RepositoryInterface; use TYPO3\Flow\Persistence\RepositoryInterface; use WE\SpreadsheetImport\Annotations\Mapping; use WE\SpreadsheetImport\Annotations\Mapping; use WE\SpreadsheetImport\Domain\Model\SpreadsheetImport; use WE\SpreadsheetImport\Domain\Model\SpreadsheetImport; /** /** * Service to handle the column mapping and the import itself * * @Flow\Scope("singleton") * @Flow\Scope("singleton") */ */ class SpreadsheetImportService { class SpreadsheetImportService { const VALIDATION_GROUPS = array('Default', 'SpreadsheetImport'); /** /** * Domain object containing the configuration the service works with * * @var SpreadsheetImport * @var SpreadsheetImport */ */ protected $spreadsheetImport; protected $spreadsheetImport; /** /** * Class name of the domain model to be imported * * @var string * @var string */ */ protected $domain; protected $domain; Loading Loading @@ -69,25 +76,59 @@ class SpreadsheetImportService { protected $validatorResolver; protected $validatorResolver; /** /** * Inverse SpreadsheetImport mapping array * Inverse array of SpreadsheetDomain mapping array property. Always use the getter function, which will process the * property in case it is not set. * * @internal * @see SpreadsheetImportService::getInverseMappingProperties() * @var array */ private $inverseMappingProperties; /** * Identifier properties of SpreadsheetDomain mapping array. Always use the getter function, which will process the * property in case it is not set. * * * @internal * @see SpreadsheetImportService::getMappingIdentifierProperties() * @var array * @var array */ */ private $inverseSpreadsheetImportMapping; private $mappingIdentifierProperties; /** /** * Identifier properties of SpreadsheetDomain argument array. Always use the getter function, which will process the * property in case it is not set. * * @internal * @see SpreadsheetImportService::getArgumentIdentifierProperties() * @var array */ private $argumentIdentifierProperties; /** * Initialize the service before usage with the object to be worked with * * @param \WE\SpreadsheetImport\Domain\Model\SpreadsheetImport $spreadsheetImport * @param \WE\SpreadsheetImport\Domain\Model\SpreadsheetImport $spreadsheetImport * * * @return $this * @return $this */ */ public function init(SpreadsheetImport $spreadsheetImport) { public function init(SpreadsheetImport $spreadsheetImport) { $this->inverseMappingProperties = array(); $this->mappingIdentifierProperties = array(); $this->argumentIdentifierProperties = array(); $this->spreadsheetImport = $spreadsheetImport; $this->spreadsheetImport = $spreadsheetImport; $this->domain = $this->settings[$spreadsheetImport->getContext()]['domain']; $context = $spreadsheetImport->getContext(); $this->domain = $this->settings[$context]['domain']; return $this; return $this; } } /** /** * Returns the annotation properties of the related domain model with the property name as key. * * Example: * array(['name'] => :Mapping, ['id'] => :Mapping) * * @return array * @return array */ */ public function getAnnotationMappingProperties() { public function getAnnotationMappingProperties() { Loading @@ -100,6 +141,8 @@ class SpreadsheetImportService { } } /** /** * Returns the total records of the spreadsheet excluding the header row * * @return array * @return array */ */ public function getTotalRecords() { public function getTotalRecords() { Loading @@ -110,6 +153,11 @@ class SpreadsheetImportService { } } /** /** * Returns the spreadsheet column as key value pair together with the heading taken of the first row * * Example: * array(['A'] => 'firstname', ['C'] => 'id') * * @return array * @return array */ */ public function getSpreadsheetColumns() { public function getSpreadsheetColumns() { Loading @@ -130,6 +178,8 @@ class SpreadsheetImportService { } } /** /** * Returns a completely mapped new domain object with values taken of the specified row in the spreadsheet. * * @param int $number * @param int $number * * * @return object * @return object Loading @@ -145,6 +195,9 @@ class SpreadsheetImportService { } } /** /** * Run the import. It inserts, updates and deletes dependent on the configuration of the initialized * SpreadsheetImport object. * * @return void * @return void */ */ public function import() { public function import() { Loading @@ -153,15 +206,14 @@ class SpreadsheetImportService { $totalDeleted = 0; $totalDeleted = 0; $processedObjectIds = array(); $processedObjectIds = array(); $objectRepository = $this->getDomainRepository(); $objectRepository = $this->getDomainRepository(); $objectValidator = $this->validatorResolver->getBaseValidatorConjunction($this->domain); $objectValidator = $this->validatorResolver->getBaseValidatorConjunction($this->domain, self::VALIDATION_GROUPS); $identifierProperties = $this->getDomainMappingIdentifierProperties(); $sheet = $this->getFileActiveSheet(); $sheet = $this->getFileActiveSheet(); $persistRecordsChunkSize = intval($this->settings['persistRecordsChunkSize']); $persistRecordsChunkSize = intval($this->settings['persistRecordsChunkSize']); $totalCount = 0; $totalCount = 0; /** @var \PHPExcel_Worksheet_Row $row */ /** @var \PHPExcel_Worksheet_Row $row */ foreach ($sheet->getRowIterator(2) as $row) { foreach ($sheet->getRowIterator(2) as $row) { $totalCount++; $totalCount++; $object = $this->findObjectByIdentifierPropertiesPerRow($identifierProperties, $row); $object = $this->findExistingObjectByRow($row); if (is_object($object)) { if (is_object($object)) { $processedObjectIds[] = $this->persistenceManager->getIdentifierByObject($object); $processedObjectIds[] = $this->persistenceManager->getIdentifierByObject($object); if ($this->spreadsheetImport->isUpdating()) { if ($this->spreadsheetImport->isUpdating()) { Loading Loading @@ -194,7 +246,7 @@ class SpreadsheetImportService { } } $deleteCount = 0; $deleteCount = 0; if ($this->spreadsheetImport->isDeleting()) { if ($this->spreadsheetImport->isDeleting()) { $notExistingObjects = $this->findObjectsByExcludedIds($processedObjectIds); $notExistingObjects = $this->findObjectsByArgumentsAndExcludedIds($processedObjectIds); foreach ($notExistingObjects as $object) { foreach ($notExistingObjects as $object) { $objectRepository->remove($object); $objectRepository->remove($object); if (++$deleteCount % $persistRecordsChunkSize === 0) { if (++$deleteCount % $persistRecordsChunkSize === 0) { Loading @@ -210,6 +262,8 @@ class SpreadsheetImportService { } } /** /** * Return the active sheet of a spreadsheet. Other potential sheets are ignored on the import. * * @return \PHPExcel_Worksheet * @return \PHPExcel_Worksheet */ */ private function getFileActiveSheet() { private function getFileActiveSheet() { Loading @@ -220,52 +274,30 @@ class SpreadsheetImportService { } } /** /** * @return \TYPO3\Flow\Persistence\RepositoryInterface * Returns an existing object found or null for a specific row. */ * private function getDomainRepository() { $repositoryClassName = preg_replace(array('/\\\Model\\\/', '/$/'), array('\\Repository\\', 'Repository'), $this->domain); /** @var RepositoryInterface $repository */ $repository = $this->objectManager->get($repositoryClassName); return $repository; } /** * @return array */ private function getDomainMappingIdentifierProperties() { // TODO: Don't use the annotation properties but the SpreadsheetImport mapping since we store the Mapping object there as well $domainMappingProperties = array(); $properties = $this->reflectionService->getPropertyNamesByAnnotation($this->domain, Mapping::class); foreach ($properties as $property) { /** @var Mapping $mapping */ $mapping = $this->reflectionService->getPropertyAnnotation($this->domain, $property, Mapping::class); if ($mapping->identifier) { $domainMappingProperties[$property] = $mapping; } } return $domainMappingProperties; } /** * @param array $identifierProperties * @param \PHPExcel_Worksheet_Row $row * @param \PHPExcel_Worksheet_Row $row * * * @return null|object * @return null|object */ */ private function findObjectByIdentifierPropertiesPerRow(array $identifierProperties, \PHPExcel_Worksheet_Row $row) { private function findExistingObjectByRow(\PHPExcel_Worksheet_Row $row) { $query = $this->getDomainRepository()->createQuery(); $query = $this->getDomainRepository()->createQuery(); $constraints = array(); $constraints = array(); $spreadsheetImportMapping = $this->spreadsheetImport->getMapping(); $mappingIdentifierProperties = $this->getMappingIdentifierProperties(); foreach ($mappingIdentifierProperties as $property => $columnMapping) { $column = $columnMapping['column']; /** @var Mapping $mapping */ /** @var Mapping $mapping */ foreach ($identifierProperties as $property => $mapping) { $mapping = $columnMapping['mapping']; $column = $spreadsheetImportMapping[$property]['column']; $propertyName = $mapping->queryPropertyName ?: $property; /** @var \PHPExcel_Worksheet_RowCellIterator $cellIterator */ /** @var \PHPExcel_Worksheet_RowCellIterator $cellIterator */ $cellIterator = $row->getCellIterator($column, $column); $cellIterator = $row->getCellIterator($column, $column); $value = $cellIterator->current()->getValue(); $value = $cellIterator->current()->getValue(); $propertyName = $mapping->queryPropertyName ?: $property; $constraints[] = $query->equals($propertyName, $value); $constraints[] = $query->equals($propertyName, $value); } } $this->mergeQueryConstraintsWithArgumentIdentifiers($query, $constraints); $argumentIdentifierProperties = $this->getArgumentIdentifierProperties(); foreach ($argumentIdentifierProperties as $property => $value) { $constraints[] = $query->equals($property, $value); } if (!empty($constraints)) { if (!empty($constraints)) { return $query->matching($query->logicalAnd($constraints))->execute()->getFirst(); return $query->matching($query->logicalAnd($constraints))->execute()->getFirst(); } else { } else { Loading @@ -274,68 +306,64 @@ class SpreadsheetImportService { } } /** /** * @param \TYPO3\Flow\Persistence\QueryInterface $query * @param array $identifiers * @param array $constraints * * @return \TYPO3\Flow\Persistence\QueryResultInterface */ */ private function mergeQueryConstraintsWithArgumentIdentifiers(QueryInterface $query, &$constraints) { private function findObjectsByArgumentsAndExcludedIds(array $identifiers) { $contextArguments = $this->settings[$this->spreadsheetImport->getContext()]['arguments']; $query = $this->getDomainRepository()->createQuery(); if (is_array($contextArguments)) { $constraints[] = $query->logicalNot($query->in('Persistence_Object_Identifier', $identifiers)); foreach ($contextArguments as $contextArgument) { // Include all the arguments into the query to get a subset only if (isset($contextArgument['identifier']) && $contextArgument['identifier'] == TRUE) { foreach ($this->spreadsheetImport->getArguments() as $name => $value) { $name = $contextArgument['name']; $arguments = $this->spreadsheetImport->getArguments(); if (array_key_exists($name, $arguments)) { $value = $arguments[$name]; $constraints[] = $query->equals($name, $value); $constraints[] = $query->equals($name, $value); } } return $query->matching($query->logicalAnd($constraints))->execute(); } } } } /** * @param object $object * @param \PHPExcel_Worksheet_Row $row */ private function setObjectPropertiesByRow($object, $row) { /* Set the argument properties before the mapping properties, as mapping property setters might be dependent on argument property values */ $this->setObjectArgumentProperties($object); $this->setObjectMappingProperties($object, $row); } } /** /** * @param \TYPO3\Flow\Persistence\QueryInterface $query * @param $object * @param array $constraints */ */ private function mergeQueryConstraintsWithArguments(QueryInterface $query, &$constraints) { private function setObjectArgumentProperties($object) { $contextArguments = $this->settings[$this->spreadsheetImport->getContext()]['arguments']; $contextArguments = $this->settings[$this->spreadsheetImport->getContext()]['arguments']; if (is_array($contextArguments)) { if (is_array($contextArguments)) { $arguments = $this->spreadsheetImport->getArguments(); foreach ($contextArguments as $contextArgument) { foreach ($contextArguments as $contextArgument) { $name = $contextArgument['name']; $name = $contextArgument['name']; $arguments = $this->spreadsheetImport->getArguments(); if (array_key_exists($name, $arguments)) { if (array_key_exists($name, $arguments)) { if (array_key_exists('domain', $contextArgument)) { $value = $this->propertyMapper->convert($arguments[$name], $contextArgument['domain']); } else { $value = $arguments[$name]; $value = $arguments[$name]; $constraints[] = $query->equals($name, $value); } } $setter = 'set' . ucfirst($name); $object->$setter($value); } } } } } } /** * @param array $identifiers * * @return \TYPO3\Flow\Persistence\QueryResultInterface */ private function findObjectsByExcludedIds(array $identifiers) { $query = $this->getDomainRepository()->createQuery(); $constraints[] = $query->logicalNot($query->in('Persistence_Object_Identifier', $identifiers)); $this->mergeQueryConstraintsWithArguments($query, $constraints); return $query->matching($query->logicalAnd($constraints))->execute(); } } /** /** * @param object $object * @param object $object * @param \PHPExcel_Worksheet_Row $row * @param \PHPExcel_Worksheet_Row $row */ */ private function setObjectPropertiesByRow($object, $row) { private function setObjectMappingProperties($object, $row) { // Set the arguments first as mapping property setters might be dependent on argument properties $inversedMappingProperties = $this->getInverseMappingProperties(); $this->setObjectArgumentProperties($object); $inverseSpreadsheetImportMapping = $this->getInverseSpreadsheetImportMapping(); /** @var \PHPExcel_Cell $cell */ /** @var \PHPExcel_Cell $cell */ foreach ($row->getCellIterator() as $cell) { foreach ($row->getCellIterator() as $cell) { $column = $cell->getColumn(); $column = $cell->getColumn(); if (array_key_exists($column, $inverseSpreadsheetImportMapping)) { if (array_key_exists($column, $inversedMappingProperties)) { $properties = $inverseSpreadsheetImportMapping[$column]; $properties = $inversedMappingProperties[$column]; foreach ($properties as $propertyMapping) { foreach ($properties as $propertyMapping) { $property = $propertyMapping['property']; $property = $propertyMapping['property']; /** @var Mapping $mapping */ /** @var Mapping $mapping */ Loading @@ -349,40 +377,84 @@ class SpreadsheetImportService { /** /** * Return an inverse SpreadsheetImport mapping array. It flips the property and column attribute and returns it as a * Return an inverse SpreadsheetImport mapping array. It flips the property and column attribute and returns it as a * 3-dim array instead of a 2-dim array. The reason for that is the case when the same column is assigned to multiple * 3-dim array instead of a 2-dim array. This is done for the case when the same column is assigned to multiple * properties. * properties. * * Example: * array( * ['A'] => array( * [0] => array(['property'] => 'name', ['mapping'] => :Mapping), * [1] => array(['property'] => 'id', ['mapping'] => :Mapping))); * * @return array */ */ private function getInverseSpreadsheetImportMapping() { private function getInverseMappingProperties() { if (empty($this->inverseSpreadsheetImportMapping)) { if (empty($this->inverseMappingProperties)) { $this->inverseSpreadsheetImportMapping = array(); $this->inverseMappingProperties = array(); foreach ($this->spreadsheetImport->getMapping() as $property => $columnMapping) { foreach ($this->spreadsheetImport->getMapping() as $property => $columnMapping) { $column = $columnMapping['column']; $column = $columnMapping['column']; $propertyMapping = array('property' => $property, 'mapping' => $columnMapping['mapping']); $propertyMapping = array('property' => $property, 'mapping' => $columnMapping['mapping']); $this->inverseSpreadsheetImportMapping[$column][] = $propertyMapping; $this->inverseMappingProperties[$column][] = $propertyMapping; } } } } return $this->inverseSpreadsheetImportMapping; return $this->inverseMappingProperties; } } /** /** * @param $object * Filters the SpreadsheetImport mappings elements for Mapping identifiers only. * * Example: * array(['id'] => array(['column'] => 'A', ['mapping'] => :Mapping))); * * @return array */ */ private function setObjectArgumentProperties($object) { private function getMappingIdentifierProperties() { $contextArguments = $this->settings[$this->spreadsheetImport->getContext()]['arguments']; if (empty($this->mappingIdentifierProperties)) { foreach ($this->spreadsheetImport->getMapping() as $property => $columnMapping) { /** @var Mapping $mapping */ $mapping = $columnMapping['mapping']; if ($mapping->identifier) { $this->mappingIdentifierProperties[$property] = $columnMapping; } } } return $this->mappingIdentifierProperties; } /** * Filters the SpreadsheetImport argument elements for identifier arguments only. * * Example: * array(['id'] => '8df83181-6fb2-11e4-8b37-00505601050c'); * * @return array */ private function getArgumentIdentifierProperties() { if (empty($this->argumentIdentifierProperties)) { $context = $this->spreadsheetImport->getContext(); $contextArguments = $this->settings[$context]['arguments']; if (is_array($contextArguments)) { if (is_array($contextArguments)) { $arguments = $this->spreadsheetImport->getArguments(); foreach ($contextArguments as $contextArgument) { foreach ($contextArguments as $contextArgument) { if (isset($contextArgument['identifier']) && $contextArgument['identifier'] == TRUE) { $name = $contextArgument['name']; $name = $contextArgument['name']; $arguments = $this->spreadsheetImport->getArguments(); if (array_key_exists($name, $arguments)) { if (array_key_exists($name, $arguments)) { if (array_key_exists('domain', $contextArgument)) { $this->argumentIdentifierProperties[$name] = $arguments[$name]; $value = $this->propertyMapper->convert($arguments[$name], $contextArgument['domain']); } } else { $value = $arguments[$name]; } } $setter = 'set' . ucfirst($name); $object->$setter($value); } } } } } } return $this->argumentIdentifierProperties; } /** * @return \TYPO3\Flow\Persistence\RepositoryInterface */ private function getDomainRepository() { $repositoryClassName = preg_replace(array('/\\\Model\\\/', '/$/'), array('\\Repository\\', 'Repository'), $this->domain); /** @var RepositoryInterface $repository */ $repository = $this->objectManager->get($repositoryClassName); return $repository; } } } }