Skip to content

W
WE.SpreadsheetImport
Commit 4269a602 authored by Chivy Lim's avatar Chivy Lim
Browse files
Options

Merge branch 'KIME-4583' into 'master' 

Bugfixes and improvements

refs KIME-4583

See merge request !15
parents 0f755ee6 eb20ee60
Hide whitespace changes
Inline Side-by-side
Showing with 516 additions and 161 deletions
Classes/WE/SpreadsheetImport/Command/SpreadsheetImportCommandController.php
View file @ 4269a602
......@@ -51,7 +51,7 @@ class SpreadsheetImportCommandController extends CommandController {
public function importCommand() {
$currentImportingCount = $this->spreadsheetImportRepository->countByImportingStatus(SpreadsheetImport::IMPORTING_STATUS_IN_PROGRESS);
if ($currentImportingCount > 0) {
$this->outputFormatted('Previous spreadsheet import is still in progress.');
$this->outputLine('Previous spreadsheet import is still in progress.');
$this->quit();
}
/** @var SpreadsheetImport $spreadsheetImport */
......@@ -67,35 +67,65 @@ class SpreadsheetImportCommandController extends CommandController {
try {
$this->spreadsheetImportService->import();
$spreadsheetImport->setImportingStatus(SpreadsheetImport::IMPORTING_STATUS_COMPLETED);
$this->outputFormatted('Spreadsheet has been imported. %d inserted, %d updated, %d deleted, %d skipped',
$this->outputLine('Spreadsheet has been imported. %d inserted, %d updated, %d deleted, %d skipped',
array($spreadsheetImport->getTotalInserted(), $spreadsheetImport->getTotalUpdated(), $spreadsheetImport->getTotalDeleted(), $spreadsheetImport->getTotalSkipped()));
} catch (Exception $e) {
} catch (\Exception $e) {
$spreadsheetImport->setImportingStatus(SpreadsheetImport::IMPORTING_STATUS_FAILED);
$this->outputFormatted('Spreadsheet import failed.');
$this->outputLine('Spreadsheet import failed.');
}
try {
$this->spreadsheetImportRepository->update($spreadsheetImport);
} catch (\Exception $e) {
$this->outputLine('Spreadsheet import status update error. It remains in progress until cleanup.');
}
$this->spreadsheetImportRepository->update($spreadsheetImport);
} else {
$this->outputFormatted('No spreadsheet import in queue.');
}
}
/**
* Cleanup previous spreadsheet imports. Threashold defined in settings.
* Cleanup past and stalled imports.
*
* @param int $keepPastImportsThreasholdDays Overwrites the setting value
* @param int $maxExecutionThreasholdMinutes Overwrites the setting value
*/
public function cleanupCommand() {
$cleanupImportsThreasholdDays = intval($this->settings['cleanupImportsThreasholdDays']);
$cleanupFromDate = new \DateTime();
$cleanupFromDate->sub(new \DateInterval('P' . $cleanupImportsThreasholdDays . 'D'));
$oldSpreadsheetImports = $this->spreadsheetImportRepository->findPreviousImportsBySpecificDate($cleanupFromDate);
if ($oldSpreadsheetImports->count() > 0) {
/** @var SpreadsheetImport $oldSpreadsheetImport */
foreach ($oldSpreadsheetImports as $oldSpreadsheetImport) {
$this->spreadsheetImportRepository->remove($oldSpreadsheetImport);
}
$this->outputLine('%d spreadsheet imports were removed.', array($oldSpreadsheetImports->count()));
} else {
$this->outputLine('There is no spreadsheet import in queue to remove.');
public function cleanupCommand($keepPastImportsThreasholdDays = -1, $maxExecutionThreasholdMinutes = -1) {
$keepPastImportsThreasholdDays = ($keepPastImportsThreasholdDays >= 0) ? $keepPastImportsThreasholdDays : intval($this->settings['keepPastImportsThreasholdDays']);
$maxExecutionThreasholdMinutes = ($maxExecutionThreasholdMinutes >= 0) ? $maxExecutionThreasholdMinutes : intval($this->settings['maxExecutionThreasholdMinutes']);
$this->cleanupPastImports($keepPastImportsThreasholdDays);
$this->cleanupStalledImports($maxExecutionThreasholdMinutes);
}
/**
* Delete past imports
*
* @param int $keepPastImportsThreasholdDays
*/
private function cleanupPastImports($keepPastImportsThreasholdDays) {
$cleanupFromDateTime = new \DateTime();
$cleanupFromDateTime->sub(new \DateInterval('P' . $keepPastImportsThreasholdDays . 'D'));
$spreadsheetImports = $this->spreadsheetImportRepository->findBySpecificDateTimeAndImportingStatus($cleanupFromDateTime);
/** @var SpreadsheetImport $spreadsheetImport */
foreach ($spreadsheetImports as $spreadsheetImport) {
$this->spreadsheetImportRepository->remove($spreadsheetImport);
}
$this->outputLine('%d spreadsheet imports removed.', array($spreadsheetImports->count()));
}
/**
* Set stalled imports to failed
*
* @param int $maxExecutionThreasholdMinutes
*/
private function cleanupStalledImports($maxExecutionThreasholdMinutes) {
$cleanupFromDateTime = new \DateTime();
$cleanupFromDateTime->sub(new \DateInterval('PT' . $maxExecutionThreasholdMinutes . 'M'));
$spreadsheetImports = $this->spreadsheetImportRepository->findBySpecificDateTimeAndImportingStatus($cleanupFromDateTime, SpreadsheetImport::IMPORTING_STATUS_IN_PROGRESS);
/** @var SpreadsheetImport $spreadsheetImport */
foreach ($spreadsheetImports as $spreadsheetImport) {
$spreadsheetImport->setImportingStatus(SpreadsheetImport::IMPORTING_STATUS_FAILED);
$this->spreadsheetImportRepository->update($spreadsheetImport);
}
$this->outputLine('%d spreadsheet imports set to failed.', array($spreadsheetImports->count()));
}
}
Classes/WE/SpreadsheetImport/Domain/Model/SpreadsheetImport.php
View file @ 4269a602
......@@ -38,10 +38,19 @@ class SpreadsheetImport {
protected $file;
/**
* DateTime when import is scheduled to progress
*
* @var \DateTime
*/
protected $scheduleDate;
/**
* Actual DateTime when import is set to in progress
*
* @var \DateTime
*/
protected $progressDate;
/**
* @var string
* @ORM\Column(type="text")
......@@ -100,6 +109,7 @@ class SpreadsheetImport {
*/
public function __construct() {
$this->scheduleDate = new \DateTime();
$this->progressDate = new \DateTime();
}
/**
......@@ -142,6 +152,7 @@ class SpreadsheetImport {
*/
public function setScheduleDate($scheduleDate) {
$this->scheduleDate = $scheduleDate;
$this->progressDate = $scheduleDate;
}
/**
......@@ -166,7 +177,11 @@ class SpreadsheetImport {
* @return array
*/
public function getArguments() {
return unserialize($this->arguments);
$arguments = unserialize($this->arguments);
if (! is_array($arguments)) {
$arguments = array();
}
return $arguments;
}
/**
......@@ -234,6 +249,9 @@ class SpreadsheetImport {
*/
public function setImportingStatus($importingStatus) {
$this->importingStatus = $importingStatus;
if ($importingStatus === self::IMPORTING_STATUS_IN_PROGRESS) {
$this->progressDate = new \DateTime();
}
}
/**
......
Classes/WE/SpreadsheetImport/Domain/Repository/SpreadsheetImportRepository.php
View file @ 4269a602
......@@ -42,12 +42,16 @@ class SpreadsheetImportRepository extends Repository {
/**
* @param \DateTime $dateTime
* @param int $importingStatus
*
* @return \TYPO3\Flow\Persistence\QueryResultInterface
*/
public function findPreviousImportsBySpecificDate(\DateTime $dateTime) {
public function findBySpecificDateTimeAndImportingStatus(\DateTime $dateTime, $importingStatus = -1) {
$query = $this->createQuery();
$constraint = $query->lessThanOrEqual('scheduleDate', $dateTime);
$constraint = $query->lessThanOrEqual('progressDate', $dateTime);
if ($importingStatus >= 0) {
$constraint = $query->logicalAnd($constraint, $query->equals('importingStatus', $importingStatus));
}
return $query->matching($constraint)->execute();
}
......
Classes/WE/SpreadsheetImport/FrontendMappingService.php
View file @ 4269a602
......@@ -122,7 +122,7 @@ class FrontendMappingService {
$previewObject = $this->spreadsheetImportService->getObjectByRow($record);
$preview = array();
$hasErrors = FALSE;
$objectValidator = $this->validatorResolver->getBaseValidatorConjunction($domain);
$objectValidator = $this->validatorResolver->getBaseValidatorConjunction($domain, SpreadsheetImportService::VALIDATION_GROUPS);
$errors = $objectValidator->validate($previewObject)->getFlattenedErrors();
foreach ($mapping as $property => $columnMapping) {
/** @var Mapping $mapping */
......
Classes/WE/SpreadsheetImport/SpreadsheetImportService.php
View file @ 4269a602
......@@ -12,22 +12,29 @@ namespace WE\SpreadsheetImport;
* */
use TYPO3\Flow\Annotations as Flow;
use TYPO3\Flow\Persistence\QueryInterface;
use TYPO3\Flow\Persistence\RepositoryInterface;
use WE\SpreadsheetImport\Annotations\Mapping;
use WE\SpreadsheetImport\Domain\Model\SpreadsheetImport;
/**
* Service to handle the column mapping and the import itself
*
* @Flow\Scope("singleton")
*/
class SpreadsheetImportService {
const VALIDATION_GROUPS = array('Default', 'SpreadsheetImport');
/**
* Domain object containing the configuration the service works with
*
* @var SpreadsheetImport
*/
protected $spreadsheetImport;
/**
* Class name of the domain model to be imported
*
* @var string
*/
protected $domain;
......@@ -69,25 +76,59 @@ class SpreadsheetImportService {
protected $validatorResolver;
/**
* Inverse SpreadsheetImport mapping array
* Inverse array of SpreadsheetDomain mapping array property. Always use the getter function, which will process the
* property in case it is not set.
*
* @internal
* @see SpreadsheetImportService::getInverseMappingProperties()
* @var array
*/
private $inverseSpreadsheetImportMapping;
private $inverseMappingProperties;
/**
* Identifier properties of SpreadsheetDomain mapping array. Always use the getter function, which will process the
* property in case it is not set.
*
* @internal
* @see SpreadsheetImportService::getMappingIdentifierProperties()
* @var array
*/
private $mappingIdentifierProperties;
/**
* Identifier properties of SpreadsheetDomain argument array. Always use the getter function, which will process the
* property in case it is not set.
*
* @internal
* @see SpreadsheetImportService::getArgumentIdentifierProperties()
* @var array
*/
private $argumentIdentifierProperties;
/**
* Initialize the service before usage with the object to be worked with
*
* @param \WE\SpreadsheetImport\Domain\Model\SpreadsheetImport $spreadsheetImport
*
* @return $this
*/
public function init(SpreadsheetImport $spreadsheetImport) {
$this->inverseMappingProperties = array();
$this->mappingIdentifierProperties = array();
$this->argumentIdentifierProperties = array();
$this->spreadsheetImport = $spreadsheetImport;
$this->domain = $this->settings[$spreadsheetImport->getContext()]['domain'];
$context = $spreadsheetImport->getContext();
$this->domain = $this->settings[$context]['domain'];
return $this;
}
/**
* Returns the annotation properties of the related domain model with the property name as key.
*
* Example:
* array(['name'] => :Mapping, ['id'] => :Mapping)
*
* @return array
*/
public function getAnnotationMappingProperties() {
......@@ -100,6 +141,8 @@ class SpreadsheetImportService {
}
/**
* Returns the total records of the spreadsheet excluding the header row
*
* @return array
*/
public function getTotalRecords() {
......@@ -110,6 +153,11 @@ class SpreadsheetImportService {
}
/**
* Returns the spreadsheet column as key value pair together with the heading taken of the first row
*
* Example:
* array(['A'] => 'firstname', ['C'] => 'id')
*
* @return array
*/
public function getSpreadsheetColumns() {
......@@ -130,6 +178,8 @@ class SpreadsheetImportService {
}
/**
* Returns a completely mapped new domain object with values taken of the specified row in the spreadsheet.
*
* @param int $number
*
* @return object
......@@ -145,6 +195,9 @@ class SpreadsheetImportService {
}
/**
* Run the import. It inserts, updates and deletes dependent on the configuration of the initialized
* SpreadsheetImport object.
*
* @return void
*/
public function import() {
......@@ -153,15 +206,14 @@ class SpreadsheetImportService {
$totalDeleted = 0;
$processedObjectIds = array();
$objectRepository = $this->getDomainRepository();
$objectValidator = $this->validatorResolver->getBaseValidatorConjunction($this->domain);
$identifierProperties = $this->getDomainMappingIdentifierProperties();
$objectValidator = $this->validatorResolver->getBaseValidatorConjunction($this->domain, self::VALIDATION_GROUPS);
$sheet = $this->getFileActiveSheet();
$persistRecordsChunkSize = intval($this->settings['persistRecordsChunkSize']);
$totalCount = 0;
/** @var \PHPExcel_Worksheet_Row $row */
foreach ($sheet->getRowIterator(2) as $row) {
$totalCount++;
$object = $this->findObjectByIdentifierPropertiesPerRow($identifierProperties, $row);
$object = $this->findExistingObjectByRow($row);
if (is_object($object)) {
$processedObjectIds[] = $this->persistenceManager->getIdentifierByObject($object);
if ($this->spreadsheetImport->isUpdating()) {
......@@ -194,7 +246,7 @@ class SpreadsheetImportService {
}
$deleteCount = 0;
if ($this->spreadsheetImport->isDeleting()) {
$notExistingObjects = $this->findObjectsByExcludedIds($processedObjectIds);
$notExistingObjects = $this->findObjectsByArgumentsAndExcludedIds($processedObjectIds);
foreach ($notExistingObjects as $object) {
$objectRepository->remove($object);
if (++$deleteCount % $persistRecordsChunkSize === 0) {
......@@ -210,6 +262,8 @@ class SpreadsheetImportService {
}
/**
* Return the active sheet of a spreadsheet. Other potential sheets are ignored on the import.
*
* @return \PHPExcel_Worksheet
*/
private function getFileActiveSheet() {
......@@ -220,52 +274,30 @@ class SpreadsheetImportService {
}
/**
* @return \TYPO3\Flow\Persistence\RepositoryInterface
*/
private function getDomainRepository() {
$repositoryClassName = preg_replace(array('/\\\Model\\\/', '/$/'), array('\\Repository\\', 'Repository'), $this->domain);
/** @var RepositoryInterface $repository */
$repository = $this->objectManager->get($repositoryClassName);
return $repository;
}
/**
* @return array
*/
private function getDomainMappingIdentifierProperties() {
// TODO: Don't use the annotation properties but the SpreadsheetImport mapping since we store the Mapping object there as well
$domainMappingProperties = array();
$properties = $this->reflectionService->getPropertyNamesByAnnotation($this->domain, Mapping::class);
foreach ($properties as $property) {
/** @var Mapping $mapping */
$mapping = $this->reflectionService->getPropertyAnnotation($this->domain, $property, Mapping::class);
if ($mapping->identifier) {
$domainMappingProperties[$property] = $mapping;
}
}
return $domainMappingProperties;
}
/**
* @param array $identifierProperties
* Returns an existing object found or null for a specific row.
*
* @param \PHPExcel_Worksheet_Row $row
*
* @return null|object
*/
private function findObjectByIdentifierPropertiesPerRow(array $identifierProperties, \PHPExcel_Worksheet_Row $row) {
private function findExistingObjectByRow(\PHPExcel_Worksheet_Row $row) {
$query = $this->getDomainRepository()->createQuery();
$constraints = array();
$spreadsheetImportMapping = $this->spreadsheetImport->getMapping();
/** @var Mapping $mapping */
foreach ($identifierProperties as $property => $mapping) {
$column = $spreadsheetImportMapping[$property]['column'];
$mappingIdentifierProperties = $this->getMappingIdentifierProperties();
foreach ($mappingIdentifierProperties as $property => $columnMapping) {
$column = $columnMapping['column'];
/** @var Mapping $mapping */
$mapping = $columnMapping['mapping'];
$propertyName = $mapping->queryPropertyName ?: $property;
/** @var \PHPExcel_Worksheet_RowCellIterator $cellIterator */
$cellIterator = $row->getCellIterator($column, $column);
$value = $cellIterator->current()->getValue();
$propertyName = $mapping->queryPropertyName ?: $property;
$constraints[] = $query->equals($propertyName, $value);
}
$this->mergeQueryConstraintsWithArgumentIdentifiers($query, $constraints);
$argumentIdentifierProperties = $this->getArgumentIdentifierProperties();
foreach ($argumentIdentifierProperties as $property => $value) {
$constraints[] = $query->equals($property, $value);
}
if (!empty($constraints)) {
return $query->matching($query->logicalAnd($constraints))->execute()->getFirst();
} else {
......@@ -274,68 +306,64 @@ class SpreadsheetImportService {
}
/**
* @param \TYPO3\Flow\Persistence\QueryInterface $query
* @param array $constraints
* @param array $identifiers
*
* @return \TYPO3\Flow\Persistence\QueryResultInterface
*/
private function mergeQueryConstraintsWithArgumentIdentifiers(QueryInterface $query, &$constraints) {
$contextArguments = $this->settings[$this->spreadsheetImport->getContext()]['arguments'];
if (is_array($contextArguments)) {
foreach ($contextArguments as $contextArgument) {
if (isset($contextArgument['identifier']) && $contextArgument['identifier'] == TRUE) {
$name = $contextArgument['name'];
$arguments = $this->spreadsheetImport->getArguments();
if (array_key_exists($name, $arguments)) {
$value = $arguments[$name];
$constraints[] = $query->equals($name, $value);
}
}
}
private function findObjectsByArgumentsAndExcludedIds(array $identifiers) {
$query = $this->getDomainRepository()->createQuery();
$constraints[] = $query->logicalNot($query->in('Persistence_Object_Identifier', $identifiers));
// Include all the arguments into the query to get a subset only
foreach ($this->spreadsheetImport->getArguments() as $name => $value) {
$constraints[] = $query->equals($name, $value);
}
return $query->matching($query->logicalAnd($constraints))->execute();
}
/**
* @param \TYPO3\Flow\Persistence\QueryInterface $query
* @param array $constraints
* @param object $object
* @param \PHPExcel_Worksheet_Row $row
*/
private function mergeQueryConstraintsWithArguments(QueryInterface $query, &$constraints) {
private function setObjectPropertiesByRow($object, $row) {
/* Set the argument properties before the mapping properties, as mapping property setters might be dependent on
argument property values */
$this->setObjectArgumentProperties($object);
$this->setObjectMappingProperties($object, $row);
}
/**
* @param $object
*/
private function setObjectArgumentProperties($object) {
$contextArguments = $this->settings[$this->spreadsheetImport->getContext()]['arguments'];
if (is_array($contextArguments)) {
$arguments = $this->spreadsheetImport->getArguments();
foreach ($contextArguments as $contextArgument) {
$name = $contextArgument['name'];
$arguments = $this->spreadsheetImport->getArguments();
if (array_key_exists($name, $arguments)) {
$value = $arguments[$name];
$constraints[] = $query->equals($name, $value);
if (array_key_exists('domain', $contextArgument)) {
$value = $this->propertyMapper->convert($arguments[$name], $contextArgument['domain']);
} else {
$value = $arguments[$name];
}
$setter = 'set' . ucfirst($name);
$object->$setter($value);
}
}
}
}
/**
* @param array $identifiers
*
* @return \TYPO3\Flow\Persistence\QueryResultInterface
*/
private function findObjectsByExcludedIds(array $identifiers) {
$query = $this->getDomainRepository()->createQuery();
$constraints[] = $query->logicalNot($query->in('Persistence_Object_Identifier', $identifiers));
$this->mergeQueryConstraintsWithArguments($query, $constraints);
return $query->matching($query->logicalAnd($constraints))->execute();
}
/**
* @param object $object
* @param \PHPExcel_Worksheet_Row $row
*/
private function setObjectPropertiesByRow($object, $row) {
// Set the arguments first as mapping property setters might be dependent on argument properties
$this->setObjectArgumentProperties($object);
$inverseSpreadsheetImportMapping = $this->getInverseSpreadsheetImportMapping();
private function setObjectMappingProperties($object, $row) {
$inversedMappingProperties = $this->getInverseMappingProperties();
/** @var \PHPExcel_Cell $cell */
foreach ($row->getCellIterator() as $cell) {
$column = $cell->getColumn();
if (array_key_exists($column, $inverseSpreadsheetImportMapping)) {
$properties = $inverseSpreadsheetImportMapping[$column];
if (array_key_exists($column, $inversedMappingProperties)) {
$properties = $inversedMappingProperties[$column];
foreach ($properties as $propertyMapping) {
$property = $propertyMapping['property'];
/** @var Mapping $mapping */
......@@ -349,40 +377,84 @@ class SpreadsheetImportService {
/**
* Return an inverse SpreadsheetImport mapping array. It flips the property and column attribute and returns it as a
* 3-dim array instead of a 2-dim array. The reason for that is the case when the same column is assigned to multiple
* 3-dim array instead of a 2-dim array. This is done for the case when the same column is assigned to multiple
* properties.
*
* Example:
* array(
* ['A'] => array(
* [0] => array(['property'] => 'name', ['mapping'] => :Mapping),
* [1] => array(['property'] => 'id', ['mapping'] => :Mapping)));
*
* @return array
*/
private function getInverseSpreadsheetImportMapping() {
if (empty($this->inverseSpreadsheetImportMapping)) {
$this->inverseSpreadsheetImportMapping = array();
private function getInverseMappingProperties() {
if (empty($this->inverseMappingProperties)) {
$this->inverseMappingProperties = array();
foreach ($this->spreadsheetImport->getMapping() as $property => $columnMapping) {
$column = $columnMapping['column'];
$propertyMapping = array('property' => $property, 'mapping' => $columnMapping['mapping']);
$this->inverseSpreadsheetImportMapping[$column][] = $propertyMapping;
$this->inverseMappingProperties[$column][] = $propertyMapping;
}
}
return $this->inverseSpreadsheetImportMapping;
return $this->inverseMappingProperties;
}
/**
* @param $object
* Filters the SpreadsheetImport mappings elements for Mapping identifiers only.
*
* Example:
* array(['id'] => array(['column'] => 'A', ['mapping'] => :Mapping)));
*
* @return array
*/
private function setObjectArgumentProperties($object) {
$contextArguments = $this->settings[$this->spreadsheetImport->getContext()]['arguments'];
if (is_array($contextArguments)) {
$arguments = $this->spreadsheetImport->getArguments();
foreach ($contextArguments as $contextArgument) {
$name = $contextArgument['name'];
if (array_key_exists($name, $arguments)) {
if (array_key_exists('domain', $contextArgument)) {
$value = $this->propertyMapper->convert($arguments[$name], $contextArgument['domain']);
} else {
$value = $arguments[$name];
private function getMappingIdentifierProperties() {
if (empty($this->mappingIdentifierProperties)) {
foreach ($this->spreadsheetImport->getMapping() as $property => $columnMapping) {
/** @var Mapping $mapping */
$mapping = $columnMapping['mapping'];
if ($mapping->identifier) {
$this->mappingIdentifierProperties[$property] = $columnMapping;
}
}
}
return $this->mappingIdentifierProperties;
}
/**
* Filters the SpreadsheetImport argument elements for identifier arguments only.
*
* Example:
* array(['id'] => '8df83181-6fb2-11e4-8b37-00505601050c');
*
* @return array
*/
private function getArgumentIdentifierProperties() {
if (empty($this->argumentIdentifierProperties)) {
$context = $this->spreadsheetImport->getContext();
$contextArguments = $this->settings[$context]['arguments'];
if (is_array($contextArguments)) {
foreach ($contextArguments as $contextArgument) {
if (isset($contextArgument['identifier']) && $contextArgument['identifier'] == TRUE) {
$name = $contextArgument['name'];
$arguments = $this->spreadsheetImport->getArguments();
if (array_key_exists($name, $arguments)) {
$this->argumentIdentifierProperties[$name] = $arguments[$name];
}
}
$setter = 'set' . ucfirst($name);
$object->$setter($value);
}
}
}
return $this->argumentIdentifierProperties;
}
/**
* @return \TYPO3\Flow\Persistence\RepositoryInterface
*/
private function getDomainRepository() {
$repositoryClassName = preg_replace(array('/\\\Model\\\/', '/$/'), array('\\Repository\\', 'Repository'), $this->domain);
/** @var RepositoryInterface $repository */
$repository = $this->objectManager->get($repositoryClassName);
return $repository;
}
}
Configuration/Settings.yaml
View file @ 4269a602
WE:
SpreadsheetImport:
cleanupImportsThreasholdDays: 365
keepPastImportsThreasholdDays: 365
maxExecutionThreasholdMinutes: 720
persistRecordsChunkSize: 100
Configuration/Settings.yaml.example
View file @ 4269a602
WE:
SpreadsheetImport:
cleanupImportsThreasholdDays: 365
keepPastImportsThreasholdDays: 365
maxExecutionThreasholdMinutes: 720
persistRecordsChunkSize: 100
default:
domain: WE\Sample\Domain\Model\User
......
Configuration/Testing/Settings.yaml
View file @ 4269a602
WE:
SpreadsheetImport:
testing:
domain: WE\SpreadsheetImport\Tests\Functional\Fixtures\ImportTarget
arguments: ~
domain: WE\SpreadsheetImport\Tests\Functional\Fixtures\Domain\Model\ImportTarget
arguments:
- { name: category, domain: 'WE\SpreadsheetImport\Tests\Functional\Fixtures\Domain\Model\ImportTargetCategory', identifier: TRUE }
- { name: comment, static: 'Sample import' }
Migrations/Mysql/Version20161103191621.php 0 → 100644
View file @ 4269a602
<?php
namespace TYPO3\Flow\Persistence\Doctrine\Migrations;
use Doctrine\DBAL\Migrations\AbstractMigration;
use Doctrine\DBAL\Schema\Schema;
/**
* Add progressdate property
*/
class Version20161103191621 extends AbstractMigration
{
/**
* @return string
*/
public function getDescription() {
return '';
}
/**
* @param Schema $schema
* @return void
*/
public function up(Schema $schema)
{
$this->abortIf($this->connection->getDatabasePlatform()->getName() != 'mysql', 'Migration can only be executed safely on "mysql".');
$this->addSql('ALTER TABLE we_spreadsheetimport_domain_model_spreadsheetimport ADD progressdate DATETIME NOT NULL');
}
/**
* @param Schema $schema
* @return void
*/
public function down(Schema $schema)
{
$this->abortIf($this->connection->getDatabasePlatform()->getName() != 'mysql', 'Migration can only be executed safely on "mysql".');
$this->addSql('ALTER TABLE we_spreadsheetimport_domain_model_spreadsheetimport DROP progressdate');
}
}
Resources/Private/Templates/SpreadsheetImport/Preview.html
View file @ 4269a602
......@@ -20,7 +20,7 @@
<f:form.hidden name="spreadsheetImport" value="{spreadsheetImport}"/>
<f:for each="{preview}" key="property" as="previewMapping">
<div>
<label>
<label class="{f:if(condition: previewMapping.error, then: 'error-text')}">
<f:if condition="{previewMapping.mapping.labelId}">
<f:then><f:translate id="{previewMapping.mapping.labelId}" />:</f:then>
<f:else>{property}:</f:else>
......
Resources/Private/Translations/de/Main.xlf
View file @ 4269a602
......@@ -41,7 +41,7 @@
<source>Import Vorschau</source>
</trans-unit>
<trans-unit id="label.spreadsheet_import.preview.invalid_record">
<source>Ungültige Daten. Dieser Record wird nicht übersprungen.</source>
<source>Ungültige Daten. Dieser Record wird übersprungen.</source>
</trans-unit>
<trans-unit id="label.spreadsheet_import.preview.record">
<source>Zeile</source>
......
Tests/Functional/Fixtures/Domain/Model/ImportTarget.php 0 → 100644
View file @ 4269a602
<?php
namespace WE\SpreadsheetImport\Tests\Functional\Fixtures\Domain\Model;
/* *
* This script belongs to the Flow package "SpreadsheetImport". *
* *
* It is free software; you can redistribute it and/or modify it under *
* the terms of the GNU Lesser General Public License, either version 3 *
* of the License, or (at your option) any later version. *
* *
* The TYPO3 project - inspiring people to share! *
* */
use TYPO3\Flow\Annotations as Flow;
use WE\SpreadsheetImport\Annotations as SpreadsheetImport;
/**
* @Flow\Entity
*/
class ImportTarget {
/**
* @var string
* @SpreadsheetImport\Mapping(identifier=true, setter="setRawId")
*/
protected $id;
/**
* @var string
* @SpreadsheetImport\Mapping
*/
protected $firstName;
/**
* @var string
* @SpreadsheetImport\Mapping
*/
protected $lastName;
/**
* @var string
* @SpreadsheetImport\Mapping
*/
protected $account;
/**
* @var ImportTargetCategory
*/
protected $category;
/**
* @var string
*/
protected $comment;
/**
* @return string
*/
public function getId() {
return $this->id;
}
/**
* @param string $id
*/
public function setId($id) {
$this->id = $id;
}
/**
* @param string $id
*/
public function setRawId($id) {
$this->id = sprintf('%05d', $id);
}
/**
* @return string
*/
public function getFirstName() {
return $this->firstName;
}
/**
* @param string $firstName
*/
public function setFirstName($firstName) {
$this->firstName = $firstName;
}
/**
* @return string
*/
public function getLastName() {
return $this->lastName;
}
/**
* @param string $lastName
*/
public function setLastName($lastName) {
$this->lastName = $lastName;
}
/**
* @return string
*/
public function getAccount() {
return $this->account;
}
/**
* @param string $account
*/
public function setAccount($account) {
$this->account = $account;
}
/**
* @return \WE\SpreadsheetImport\Tests\Functional\Fixtures\Domain\Model\ImportTargetCategory
*/
public function getCategory() {
return $this->category;
}
/**
* @param \WE\SpreadsheetImport\Tests\Functional\Fixtures\Domain\Model\ImportTargetCategory $category
*/
public function setCategory($category) {
$this->category = $category;
}
/**
* @return string
*/
public function getComment() {
return $this->comment;
}
/**
* @param string $comment
*/
public function setComment($comment) {
$this->comment = $comment;
}
}
Tests/Functional/Fixtures/ImportTarget.php → Tests/Functional/Fixtures/Domain/Model/ImportTargetCategory.php
View file @ 4269a602
<?php
namespace WE\SpreadsheetImport\Tests\Functional\Fixtures;
namespace WE\SpreadsheetImport\Tests\Functional\Fixtures\Domain\Model;
/* *
* This script belongs to the Flow package "SpreadsheetImport". *
......@@ -13,22 +13,19 @@ namespace WE\SpreadsheetImport\Tests\Functional\Fixtures;
use TYPO3\Flow\Annotations as Flow;
use WE\SpreadsheetImport\Annotations as SpreadsheetImport;
use WE\SpreadsheetImport\Domain\Model\SpreadsheetImportTargetInterface;
/**
* @Flow\Entity
*/
class ImportTarget {
class ImportTargetCategory {
/**
* @var string
* @SpreadsheetImport\Mapping(identifier=true, setter="setRawId")
*/
protected $id;
/**
* @var string
* @SpreadsheetImport\Mapping
*/
protected $name;
......@@ -46,13 +43,6 @@ class ImportTarget {
$this->id = $id;
}
/**
* @param string $id
*/
public function setRawId($id) {
$this->id = sprintf('%05d', $id);
}
/**
* @return string
*/
......@@ -66,4 +56,5 @@ class ImportTarget {
public function setName($name) {
$this->name = $name;
}
}
Tests/Functional/Fixtures/Domain/Repository/ImportTargetRepository.php 0 → 100644
View file @ 4269a602
<?php
namespace WE\SpreadsheetImport\Tests\Functional\Fixtures\Domain\Repository;
/* *
* This script belongs to the Flow package "SpreadsheetImport". *
* *
* It is free software; you can redistribute it and/or modify it under *
* the terms of the GNU Lesser General Public License, either version 3 *
* of the License, or (at your option) any later version. *
* *
* The TYPO3 project - inspiring people to share! *
* */
use TYPO3\Flow\Annotations as Flow;
use TYPO3\Flow\Persistence\Repository;
/**
* @Flow\Scope("singleton")
*/
class ImportTargetRepository extends Repository {
}
Tests/Functional/SpreadsheetImportServiceTest.php
View file @ 4269a602
......@@ -17,7 +17,8 @@ use TYPO3\Flow\Tests\FunctionalTestCase;
use WE\SpreadsheetImport\Annotations\Mapping;
use WE\SpreadsheetImport\Domain\Model\SpreadsheetImport;
use WE\SpreadsheetImport\SpreadsheetImportService;
use WE\SpreadsheetImport\Tests\Functional\Fixtures\ImportTarget;
use WE\SpreadsheetImport\Tests\Functional\Fixtures\Domain\Model\ImportTarget;
use WE\SpreadsheetImport\Tests\Functional\Fixtures\Domain\Model\ImportTargetCategory;
class SpreadsheetImportServiceTest extends FunctionalTestCase {
......@@ -41,19 +42,17 @@ class SpreadsheetImportServiceTest extends FunctionalTestCase {
$reflectionService = $this->objectManager->get(ReflectionService::class);
$this->inject($this->spreadsheetImportService, 'reflectionService', $reflectionService);
$this->initializeSpreadsheetMock();
}
/**
* @return void
*/
public function tearDown() {
$persistenceManager = self::$bootstrap->getObjectManager()->get('TYPO3\Flow\Persistence\PersistenceManagerInterface');
$persistenceManager = $this->objectManager->get('TYPO3\Flow\Persistence\PersistenceManagerInterface');
if (is_callable(array($persistenceManager, 'tearDown'))) {
$persistenceManager->tearDown();
}
self::$bootstrap->getObjectManager()->forgetInstance('TYPO3\Flow\Persistence\PersistenceManagerInterface');
$this->objectManager->forgetInstance('TYPO3\Flow\Persistence\PersistenceManagerInterface');
parent::tearDown();
}
......@@ -62,47 +61,75 @@ class SpreadsheetImportServiceTest extends FunctionalTestCase {
$spreadsheetImport->setContext('testing');
$resource = $this->resourceManager->importResource(__DIR__ . '/Fixtures/Resources/sample.xlsx');
$spreadsheetImport->setFile($resource);
$idMapping = array('column' => 'C', 'mapping' => new Mapping());
$nameMapping = array('column' => 'A', 'mapping' => new Mapping());
$spreadsheetImport->setMapping(array('id' => $idMapping, 'name' => $nameMapping));
$this->spreadsheetImportService->init($spreadsheetImport);
return $spreadsheetImport;
}
private function initializeConfiguredSpreadsheetMock() {
$spreadsheetImport = $this->initializeSpreadsheetMock();
$annotationMappings = $this->spreadsheetImportService->getAnnotationMappingProperties();
$spreadsheetImport->setMapping(array(
'id' => array('column' => 'C', 'mapping' => $annotationMappings['id']),
'firstName' => array('column' => 'A', 'mapping' => $annotationMappings['firstName']),
'lastName' => array('column' => 'B', 'mapping' => $annotationMappings['lastName']),
'account' => array('column' => 'C', 'mapping' => $annotationMappings['account'])));
$spreadsheetImport->setArguments(array(
'category' => new ImportTargetCategory(), // Could also simply assign the UUID
'comment' => 'Sample import'
));
}
/**
* @test
*/
public function getMappingPropertiesReturnsPropertiesWithMappingAnnotation() {
public function getAnnotationMappingPropertiesReturnsArrayMappingAnnotation() {
$this->initializeSpreadsheetMock();
$properties = $this->spreadsheetImportService->getAnnotationMappingProperties();
$this->assertArrayHasKey('id', $properties);
$this->assertArrayHasKey('name', $properties);
$this->assertArrayHasKey('firstName', $properties);
/** @var Mapping $id */
$id = $properties['id'];
/** @var Mapping $name */
$name = $properties['name'];
/** @var Mapping $firstName */
$firstName = $properties['firstName'];
$this->assertSame(TRUE, $id->identifier);
$this->assertSame(FALSE, $name->identifier);
$this->assertSame(FALSE, $firstName->identifier);
}
/**
* @test
*/
public function getTotalRecordsRecordsNumberOfObjectsToImport() {
$this->initializeSpreadsheetMock();
$this->assertSame(2, $this->spreadsheetImportService->getTotalRecords());
}
/**
* @test
*/
public function getSpreadsheetColumnsReturnsColumnsWithHeadings() {
$this->initializeSpreadsheetMock();
$columns = $this->spreadsheetImportService->getSpreadsheetColumns();
$this->assertArrayHasKey('A', $columns);
$this->assertArrayHasKey('B', $columns);
$this->assertArrayHasKey('C', $columns);
$this->assertContains('name', $columns);
$this->assertContains('lastname', $columns);
$this->assertContains('id', $columns);
$this->assertSame('firstname', $columns['A']);
$this->assertSame('name', $columns['B']);
$this->assertSame('id', $columns['C']);
}
/**
* @test
*/
public function getObjectByRowOneReturnsImportTargetWithSetProperties() {
$this->initializeConfiguredSpreadsheetMock();
/** @var ImportTarget $object */
$object = $this->spreadsheetImportService->getObjectByRow(1);
$this->assertEquals('00001', $object->getId());
$this->assertEquals('Hans', $object->getName());
$this->assertSame('00001', $object->getId());
$this->assertSame('Hans', $object->getFirstName());
$this->assertSame('Muster', $object->getLastName());
$this->assertSame('001', $object->getAccount());
$this->assertInstanceOf(ImportTargetCategory::class, $object->getCategory());
$this->assertSame('Sample import', $object->getComment());
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment