Merge branch 'KIME-4583' into 'master'

Bugfixes and improvements

refs KIME-4583

See merge request !15

Classes/WE/SpreadsheetImport/Command/SpreadsheetImportCommandController.php

@@ -51,7 +51,7 @@ class SpreadsheetImportCommandController extends CommandController {
 	public function importCommand() {
 		$currentImportingCount = $this->spreadsheetImportRepository->countByImportingStatus(SpreadsheetImport::IMPORTING_STATUS_IN_PROGRESS);
 		if ($currentImportingCount > 0) {
-			$this->outputFormatted('Previous spreadsheet import is still in progress.');
+			$this->outputLine('Previous spreadsheet import is still in progress.');
 			$this->quit();
 		}
 		/** @var SpreadsheetImport $spreadsheetImport */
@@ -67,35 +67,65 @@ class SpreadsheetImportCommandController extends CommandController {
 			try {
 				$this->spreadsheetImportService->import();
 				$spreadsheetImport->setImportingStatus(SpreadsheetImport::IMPORTING_STATUS_COMPLETED);
-				$this->outputFormatted('Spreadsheet has been imported. %d inserted, %d updated, %d deleted, %d skipped',
+				$this->outputLine('Spreadsheet has been imported. %d inserted, %d updated, %d deleted, %d skipped',
 					array($spreadsheetImport->getTotalInserted(), $spreadsheetImport->getTotalUpdated(), $spreadsheetImport->getTotalDeleted(), $spreadsheetImport->getTotalSkipped()));
-			} catch (Exception $e) {
+			} catch (\Exception $e) {
 				$spreadsheetImport->setImportingStatus(SpreadsheetImport::IMPORTING_STATUS_FAILED);
-				$this->outputFormatted('Spreadsheet import failed.');
+				$this->outputLine('Spreadsheet import failed.');
 			}
+			try {
 				$this->spreadsheetImportRepository->update($spreadsheetImport);
+			} catch (\Exception $e) {
+				$this->outputLine('Spreadsheet import status update error. It remains in progress until cleanup.');
+			}
 		} else {
 			$this->outputFormatted('No spreadsheet import in queue.');
 		}
 	}
 
 	/**
-	 * Cleanup previous spreadsheet imports. Threashold defined in settings.
+	 * Cleanup past and stalled imports.
+	 *
+	 * @param int $keepPastImportsThreasholdDays Overwrites the setting value
+	 * @param int $maxExecutionThreasholdMinutes Overwrites the setting value
 	 */
-	public function cleanupCommand() {
-		$cleanupImportsThreasholdDays = intval($this->settings['cleanupImportsThreasholdDays']);
-		$cleanupFromDate = new \DateTime();
-		$cleanupFromDate->sub(new \DateInterval('P' . $cleanupImportsThreasholdDays . 'D'));
-		$oldSpreadsheetImports = $this->spreadsheetImportRepository->findPreviousImportsBySpecificDate($cleanupFromDate);
-		if ($oldSpreadsheetImports->count() > 0) {
-			/** @var SpreadsheetImport $oldSpreadsheetImport */
-			foreach ($oldSpreadsheetImports as $oldSpreadsheetImport) {
-				$this->spreadsheetImportRepository->remove($oldSpreadsheetImport);
+	public function cleanupCommand($keepPastImportsThreasholdDays = -1, $maxExecutionThreasholdMinutes = -1) {
+		$keepPastImportsThreasholdDays = ($keepPastImportsThreasholdDays >= 0) ? $keepPastImportsThreasholdDays : intval($this->settings['keepPastImportsThreasholdDays']);
+		$maxExecutionThreasholdMinutes = ($maxExecutionThreasholdMinutes >= 0) ? $maxExecutionThreasholdMinutes : intval($this->settings['maxExecutionThreasholdMinutes']);
+		$this->cleanupPastImports($keepPastImportsThreasholdDays);
+		$this->cleanupStalledImports($maxExecutionThreasholdMinutes);
 	}
-			$this->outputLine('%d spreadsheet imports were removed.', array($oldSpreadsheetImports->count()));
-		} else {
-			$this->outputLine('There is no spreadsheet import in queue to remove.');
+
+	/**
+	 * Delete past imports
+	 *
+	 * @param int $keepPastImportsThreasholdDays
+	 */
+	private function cleanupPastImports($keepPastImportsThreasholdDays) {
+		$cleanupFromDateTime = new \DateTime();
+		$cleanupFromDateTime->sub(new \DateInterval('P' . $keepPastImportsThreasholdDays . 'D'));
+		$spreadsheetImports = $this->spreadsheetImportRepository->findBySpecificDateTimeAndImportingStatus($cleanupFromDateTime);
+		/** @var SpreadsheetImport $spreadsheetImport */
+		foreach ($spreadsheetImports as $spreadsheetImport) {
+			$this->spreadsheetImportRepository->remove($spreadsheetImport);
 		}
+		$this->outputLine('%d spreadsheet imports removed.', array($spreadsheetImports->count()));
 	}
 
+	/**
+	 * Set stalled imports to failed
+	 *
+	 * @param int $maxExecutionThreasholdMinutes
+	 */
+	private function cleanupStalledImports($maxExecutionThreasholdMinutes) {
+		$cleanupFromDateTime = new \DateTime();
+		$cleanupFromDateTime->sub(new \DateInterval('PT' . $maxExecutionThreasholdMinutes . 'M'));
+		$spreadsheetImports = $this->spreadsheetImportRepository->findBySpecificDateTimeAndImportingStatus($cleanupFromDateTime, SpreadsheetImport::IMPORTING_STATUS_IN_PROGRESS);
+		/** @var SpreadsheetImport $spreadsheetImport */
+		foreach ($spreadsheetImports as $spreadsheetImport) {
+			$spreadsheetImport->setImportingStatus(SpreadsheetImport::IMPORTING_STATUS_FAILED);
+			$this->spreadsheetImportRepository->update($spreadsheetImport);
+		}
+		$this->outputLine('%d spreadsheet imports set to failed.', array($spreadsheetImports->count()));
+	}
 }

Classes/WE/SpreadsheetImport/Domain/Model/SpreadsheetImport.php

@@ -38,10 +38,19 @@ class SpreadsheetImport {
 	protected $file;
 
 	/**
+	 * DateTime when import is scheduled to progress
+	 *
 	 * @var \DateTime
 	 */
 	protected $scheduleDate;
 
+	/**
+	 * Actual DateTime when import is set to in progress
+	 *
+	 * @var \DateTime
+	 */
+	protected $progressDate;
+
 	/**
 	 * @var string
 	 * @ORM\Column(type="text")
@@ -100,6 +109,7 @@ class SpreadsheetImport {
 	 */
 	public function __construct() {
 		$this->scheduleDate = new \DateTime();
+		$this->progressDate = new \DateTime();
 	}
 
 	/**
@@ -142,6 +152,7 @@ class SpreadsheetImport {
 	 */
 	public function setScheduleDate($scheduleDate) {
 		$this->scheduleDate = $scheduleDate;
+		$this->progressDate = $scheduleDate;
 	}
 
 	/**
@@ -166,7 +177,11 @@ class SpreadsheetImport {
 	 * @return array
 	 */
 	public function getArguments() {
-		return unserialize($this->arguments);
+		$arguments = unserialize($this->arguments);
+		if (! is_array($arguments)) {
+			$arguments = array();
+		}
+		return $arguments;
 	}
 
 	/**
@@ -234,6 +249,9 @@ class SpreadsheetImport {
 	 */
 	public function setImportingStatus($importingStatus) {
 		$this->importingStatus = $importingStatus;
+		if ($importingStatus === self::IMPORTING_STATUS_IN_PROGRESS) {
+			$this->progressDate = new \DateTime();
+		}
 	}
 
 	/**

Classes/WE/SpreadsheetImport/Domain/Repository/SpreadsheetImportRepository.php

@@ -42,12 +42,16 @@ class SpreadsheetImportRepository extends Repository {
 
 	/**
 	 * @param \DateTime $dateTime
+	 * @param int $importingStatus
 	 *
 	 * @return \TYPO3\Flow\Persistence\QueryResultInterface
 	 */
-	public function findPreviousImportsBySpecificDate(\DateTime $dateTime) {
+	public function findBySpecificDateTimeAndImportingStatus(\DateTime $dateTime, $importingStatus = -1) {
 		$query = $this->createQuery();
-		$constraint = $query->lessThanOrEqual('scheduleDate', $dateTime);
+		$constraint = $query->lessThanOrEqual('progressDate', $dateTime);
+		if ($importingStatus >= 0) {
+			$constraint = $query->logicalAnd($constraint, $query->equals('importingStatus', $importingStatus));
+		}
 		return $query->matching($constraint)->execute();
 	}
 

Classes/WE/SpreadsheetImport/FrontendMappingService.php

@@ -122,7 +122,7 @@ class FrontendMappingService {
 		$previewObject = $this->spreadsheetImportService->getObjectByRow($record);
 		$preview = array();
 		$hasErrors = FALSE;
-		$objectValidator = $this->validatorResolver->getBaseValidatorConjunction($domain);
+		$objectValidator = $this->validatorResolver->getBaseValidatorConjunction($domain, SpreadsheetImportService::VALIDATION_GROUPS);
 		$errors = $objectValidator->validate($previewObject)->getFlattenedErrors();
 		foreach ($mapping as $property => $columnMapping) {
 			/** @var Mapping $mapping */

Classes/WE/SpreadsheetImport/SpreadsheetImportService.php

@@ -12,22 +12,29 @@ namespace WE\SpreadsheetImport;
  *                                                                        */
 
 use TYPO3\Flow\Annotations as Flow;
-use TYPO3\Flow\Persistence\QueryInterface;
 use TYPO3\Flow\Persistence\RepositoryInterface;
 use WE\SpreadsheetImport\Annotations\Mapping;
 use WE\SpreadsheetImport\Domain\Model\SpreadsheetImport;
 
 /**
+ * Service to handle the column mapping and the import itself
+ *
  * @Flow\Scope("singleton")
  */
 class SpreadsheetImportService {
 
+	const VALIDATION_GROUPS = array('Default', 'SpreadsheetImport');
+
 	/**
+	 * Domain object containing the configuration the service works with
+	 *
 	 * @var SpreadsheetImport
 	 */
 	protected $spreadsheetImport;
 
 	/**
+	 * Class name of the domain model to be imported
+	 *
 	 * @var string
 	 */
 	protected $domain;
@@ -69,25 +76,59 @@ class SpreadsheetImportService {
 	protected $validatorResolver;
 
 	/**
-	 * Inverse SpreadsheetImport mapping array
+	 * Inverse array of SpreadsheetDomain mapping array property. Always use the getter function, which will process the
+	 * property in case it is not set.
+	 *
+	 * @internal
+	 * @see SpreadsheetImportService::getInverseMappingProperties()
+	 * @var array
+	 */
+	private $inverseMappingProperties;
+
+	/**
+	 * Identifier properties of SpreadsheetDomain mapping array. Always use the getter function, which will process the
+	 * property in case it is not set.
 	 *
+	 * @internal
+	 * @see SpreadsheetImportService::getMappingIdentifierProperties()
 	 * @var array
 	 */
-	private $inverseSpreadsheetImportMapping;
+	private $mappingIdentifierProperties;
 
 	/**
+	 * Identifier properties of SpreadsheetDomain argument array. Always use the getter function, which will process the
+	 * property in case it is not set.
+	 *
+	 * @internal
+	 * @see SpreadsheetImportService::getArgumentIdentifierProperties()
+	 * @var array
+	 */
+	private $argumentIdentifierProperties;
+
+	/**
+	 * Initialize the service before usage with the object to be worked with
+	 *
 	 * @param \WE\SpreadsheetImport\Domain\Model\SpreadsheetImport $spreadsheetImport
 	 *
 	 * @return $this
 	 */
 	public function init(SpreadsheetImport $spreadsheetImport) {
+		$this->inverseMappingProperties = array();
+		$this->mappingIdentifierProperties = array();
+		$this->argumentIdentifierProperties = array();
 		$this->spreadsheetImport = $spreadsheetImport;
-		$this->domain = $this->settings[$spreadsheetImport->getContext()]['domain'];
+		$context = $spreadsheetImport->getContext();
+		$this->domain = $this->settings[$context]['domain'];
 
 		return $this;
 	}
 
 	/**
+	 * Returns the annotation properties of the related domain model with the property name as key.
+	 *
+	 * Example:
+	 *  array(['name'] => :Mapping, ['id'] => :Mapping)
+	 *
 	 * @return array
 	 */
 	public function getAnnotationMappingProperties() {
@@ -100,6 +141,8 @@ class SpreadsheetImportService {
 	}
 
 	/**
+	 * Returns the total records of the spreadsheet excluding the header row
+	 *
 	 * @return array
 	 */
 	public function getTotalRecords() {
@@ -110,6 +153,11 @@ class SpreadsheetImportService {
 	}
 
 	/**
+	 * Returns the spreadsheet column as key value pair together with the heading taken of the first row
+	 *
+	 * Example:
+	 *  array(['A'] => 'firstname', ['C'] => 'id')
+	 *
 	 * @return array
 	 */
 	public function getSpreadsheetColumns() {
@@ -130,6 +178,8 @@ class SpreadsheetImportService {
 	}
 
 	/**
+	 * Returns a completely mapped new domain object with values taken of the specified row in the spreadsheet.
+	 *
 	 * @param int $number
 	 *
 	 * @return object
@@ -145,6 +195,9 @@ class SpreadsheetImportService {
 	}
 
 	/**
+	 * Run the import. It inserts, updates and deletes dependent on the configuration of the initialized
+	 * SpreadsheetImport object.
+	 *
 	 * @return void
 	 */
 	public function import() {
@@ -153,15 +206,14 @@ class SpreadsheetImportService {
 		$totalDeleted = 0;
 		$processedObjectIds = array();
 		$objectRepository = $this->getDomainRepository();
-		$objectValidator = $this->validatorResolver->getBaseValidatorConjunction($this->domain);
-		$identifierProperties = $this->getDomainMappingIdentifierProperties();
+		$objectValidator = $this->validatorResolver->getBaseValidatorConjunction($this->domain, self::VALIDATION_GROUPS);
 		$sheet = $this->getFileActiveSheet();
 		$persistRecordsChunkSize = intval($this->settings['persistRecordsChunkSize']);
 		$totalCount = 0;
 		/** @var \PHPExcel_Worksheet_Row $row */
 		foreach ($sheet->getRowIterator(2) as $row) {
 			$totalCount++;
-			$object = $this->findObjectByIdentifierPropertiesPerRow($identifierProperties, $row);
+			$object = $this->findExistingObjectByRow($row);
 			if (is_object($object)) {
 				$processedObjectIds[] = $this->persistenceManager->getIdentifierByObject($object);
 				if ($this->spreadsheetImport->isUpdating()) {
@@ -194,7 +246,7 @@ class SpreadsheetImportService {
 		}
 		$deleteCount = 0;
 		if ($this->spreadsheetImport->isDeleting()) {
-			$notExistingObjects = $this->findObjectsByExcludedIds($processedObjectIds);
+			$notExistingObjects = $this->findObjectsByArgumentsAndExcludedIds($processedObjectIds);
 			foreach ($notExistingObjects as $object) {
 				$objectRepository->remove($object);
 				if (++$deleteCount % $persistRecordsChunkSize === 0) {
@@ -210,6 +262,8 @@ class SpreadsheetImportService {
 	}
 
 	/**
+	 * Return the active sheet of a spreadsheet. Other potential sheets are ignored on the import.
+	 *
 	 * @return \PHPExcel_Worksheet
 	 */
 	private function getFileActiveSheet() {
@@ -220,52 +274,30 @@ class SpreadsheetImportService {
 	}
 
 	/**
-	 * @return \TYPO3\Flow\Persistence\RepositoryInterface
-	 */
-	private function getDomainRepository() {
-		$repositoryClassName = preg_replace(array('/\\\Model\\\/', '/$/'), array('\\Repository\\', 'Repository'), $this->domain);
-		/** @var RepositoryInterface $repository */
-		$repository = $this->objectManager->get($repositoryClassName);
-		return $repository;
-	}
-
-	/**
-	 * @return array
-	 */
-	private function getDomainMappingIdentifierProperties() {
-		// TODO: Don't use the annotation properties but the SpreadsheetImport mapping since we store the Mapping object there as well
-		$domainMappingProperties = array();
-		$properties = $this->reflectionService->getPropertyNamesByAnnotation($this->domain, Mapping::class);
-		foreach ($properties as $property) {
-			/** @var Mapping $mapping */
-			$mapping = $this->reflectionService->getPropertyAnnotation($this->domain, $property, Mapping::class);
-			if ($mapping->identifier) {
-				$domainMappingProperties[$property] = $mapping;
-			}
-		}
-		return $domainMappingProperties;
-	}
-
-	/**
-	 * @param array $identifierProperties
+	 * Returns an existing object found or null for a specific row.
+	 *
 	 * @param \PHPExcel_Worksheet_Row $row
 	 *
 	 * @return null|object
 	 */
-	private function findObjectByIdentifierPropertiesPerRow(array $identifierProperties, \PHPExcel_Worksheet_Row $row) {
+	private function findExistingObjectByRow(\PHPExcel_Worksheet_Row $row) {
 		$query = $this->getDomainRepository()->createQuery();
 		$constraints = array();
-		$spreadsheetImportMapping = $this->spreadsheetImport->getMapping();
+		$mappingIdentifierProperties = $this->getMappingIdentifierProperties();
+		foreach ($mappingIdentifierProperties as $property => $columnMapping) {
+			$column = $columnMapping['column'];
 			/** @var Mapping $mapping */
-		foreach ($identifierProperties as $property => $mapping) {
-			$column = $spreadsheetImportMapping[$property]['column'];
+			$mapping = $columnMapping['mapping'];
+			$propertyName = $mapping->queryPropertyName ?: $property;
 			/** @var \PHPExcel_Worksheet_RowCellIterator $cellIterator */
 			$cellIterator = $row->getCellIterator($column, $column);
 			$value = $cellIterator->current()->getValue();
-			$propertyName = $mapping->queryPropertyName ?: $property;
 			$constraints[] = $query->equals($propertyName, $value);
 		}
-		$this->mergeQueryConstraintsWithArgumentIdentifiers($query, $constraints);
+		$argumentIdentifierProperties = $this->getArgumentIdentifierProperties();
+		foreach ($argumentIdentifierProperties as $property => $value) {
+			$constraints[] = $query->equals($property, $value);
+		}
 		if (!empty($constraints)) {
 			return $query->matching($query->logicalAnd($constraints))->execute()->getFirst();
 		} else {
@@ -274,68 +306,64 @@ class SpreadsheetImportService {
 	}
 
 	/**
-	 * @param \TYPO3\Flow\Persistence\QueryInterface $query
-	 * @param array $constraints
+	 * @param array $identifiers
+	 *
+	 * @return \TYPO3\Flow\Persistence\QueryResultInterface
 	 */
-	private function mergeQueryConstraintsWithArgumentIdentifiers(QueryInterface $query, &$constraints) {
-		$contextArguments = $this->settings[$this->spreadsheetImport->getContext()]['arguments'];
-		if (is_array($contextArguments)) {
-			foreach ($contextArguments as $contextArgument) {
-				if (isset($contextArgument['identifier']) && $contextArgument['identifier'] == TRUE) {
-					$name = $contextArgument['name'];
-					$arguments = $this->spreadsheetImport->getArguments();
-					if (array_key_exists($name, $arguments)) {
-						$value = $arguments[$name];
+	private function findObjectsByArgumentsAndExcludedIds(array $identifiers) {
+		$query = $this->getDomainRepository()->createQuery();
+		$constraints[] = $query->logicalNot($query->in('Persistence_Object_Identifier', $identifiers));
+		// Include all the arguments into the query to get a subset only
+		foreach ($this->spreadsheetImport->getArguments() as $name => $value) {
 			$constraints[] = $query->equals($name, $value);
 		}
+		return $query->matching($query->logicalAnd($constraints))->execute();
 	}
-			}
-		}
+
+	/**
+	 * @param object $object
+	 * @param \PHPExcel_Worksheet_Row $row
+	 */
+	private function setObjectPropertiesByRow($object, $row) {
+		/* Set the argument properties before the mapping properties, as mapping property setters might be dependent on
+		argument property values */
+		$this->setObjectArgumentProperties($object);
+		$this->setObjectMappingProperties($object, $row);
 	}
 
 	/**
-	 * @param \TYPO3\Flow\Persistence\QueryInterface $query
-	 * @param array $constraints
+	 * @param $object
 	 */
-	private function mergeQueryConstraintsWithArguments(QueryInterface $query, &$constraints) {
+	private function setObjectArgumentProperties($object) {
 		$contextArguments = $this->settings[$this->spreadsheetImport->getContext()]['arguments'];
 		if (is_array($contextArguments)) {
+			$arguments = $this->spreadsheetImport->getArguments();
 			foreach ($contextArguments as $contextArgument) {
 				$name = $contextArgument['name'];
-				$arguments = $this->spreadsheetImport->getArguments();
 				if (array_key_exists($name, $arguments)) {
+					if (array_key_exists('domain', $contextArgument)) {
+						$value = $this->propertyMapper->convert($arguments[$name], $contextArgument['domain']);
+					} else {
 						$value = $arguments[$name];
-					$constraints[] = $query->equals($name, $value);
 					}
+					$setter = 'set' . ucfirst($name);
+					$object->$setter($value);
 				}
 			}
 		}
-
-	/**
-	 * @param array $identifiers
-	 *
-	 * @return \TYPO3\Flow\Persistence\QueryResultInterface
-	 */
-	private function findObjectsByExcludedIds(array $identifiers) {
-		$query = $this->getDomainRepository()->createQuery();
-		$constraints[] = $query->logicalNot($query->in('Persistence_Object_Identifier', $identifiers));
-		$this->mergeQueryConstraintsWithArguments($query, $constraints);
-		return $query->matching($query->logicalAnd($constraints))->execute();
 	}
 
 	/**
 	 * @param object $object
 	 * @param \PHPExcel_Worksheet_Row $row
 	 */
-	private function setObjectPropertiesByRow($object, $row) {
-		// Set the arguments first as mapping property setters might be dependent on argument properties
-		$this->setObjectArgumentProperties($object);
-		$inverseSpreadsheetImportMapping = $this->getInverseSpreadsheetImportMapping();
+	private function setObjectMappingProperties($object, $row) {
+		$inversedMappingProperties = $this->getInverseMappingProperties();
 		/** @var \PHPExcel_Cell $cell */
 		foreach ($row->getCellIterator() as $cell) {
 			$column = $cell->getColumn();
-			if (array_key_exists($column, $inverseSpreadsheetImportMapping)) {
-				$properties = $inverseSpreadsheetImportMapping[$column];
+			if (array_key_exists($column, $inversedMappingProperties)) {
+				$properties = $inversedMappingProperties[$column];
 				foreach ($properties as $propertyMapping) {
 					$property = $propertyMapping['property'];
 					/** @var Mapping $mapping */
@@ -349,40 +377,84 @@ class SpreadsheetImportService {
 
 	/**
 	 * Return an inverse SpreadsheetImport mapping array. It flips the property and column attribute and returns it as a
-	 * 3-dim array instead of a 2-dim array. The reason for that is the case when the same column is assigned to multiple
+	 * 3-dim array instead of a 2-dim array. This is done for the case when the same column is assigned to multiple
 	 * properties.
+	 *
+	 * Example:
+	 *  array(
+	 *      ['A'] => array(
+	 *          [0] => array(['property'] => 'name', ['mapping'] => :Mapping),
+	 *          [1] => array(['property'] => 'id', ['mapping'] => :Mapping)));
+	 *
+	 * @return array
 	 */
-	private function getInverseSpreadsheetImportMapping() {
-		if (empty($this->inverseSpreadsheetImportMapping)) {
-			$this->inverseSpreadsheetImportMapping = array();
+	private function getInverseMappingProperties() {
+		if (empty($this->inverseMappingProperties)) {
+			$this->inverseMappingProperties = array();
 			foreach ($this->spreadsheetImport->getMapping() as $property => $columnMapping) {
 				$column = $columnMapping['column'];
 				$propertyMapping = array('property' => $property, 'mapping' => $columnMapping['mapping']);
-				$this->inverseSpreadsheetImportMapping[$column][] = $propertyMapping;
+				$this->inverseMappingProperties[$column][] = $propertyMapping;
 			}
 		}
-		return $this->inverseSpreadsheetImportMapping;
+		return $this->inverseMappingProperties;
 	}
 
 	/**
-	 * @param $object
+	 * Filters the SpreadsheetImport mappings elements for Mapping identifiers only.
+	 *
+	 * Example:
+	 *  array(['id'] => array(['column'] => 'A', ['mapping'] => :Mapping)));
+	 *
+	 * @return array
 	 */
-	private function setObjectArgumentProperties($object) {
-		$contextArguments = $this->settings[$this->spreadsheetImport->getContext()]['arguments'];
+	private function getMappingIdentifierProperties() {
+		if (empty($this->mappingIdentifierProperties)) {
+			foreach ($this->spreadsheetImport->getMapping() as $property => $columnMapping) {
+				/** @var Mapping $mapping */
+				$mapping = $columnMapping['mapping'];
+				if ($mapping->identifier) {
+					$this->mappingIdentifierProperties[$property] = $columnMapping;
+				}
+			}
+		}
+		return $this->mappingIdentifierProperties;
+	}
+
+	/**
+	 * Filters the SpreadsheetImport argument elements for identifier arguments only.
+	 *
+	 * Example:
+	 *  array(['id'] => '8df83181-6fb2-11e4-8b37-00505601050c');
+	 *
+	 * @return array
+	 */
+	private function getArgumentIdentifierProperties() {
+		if (empty($this->argumentIdentifierProperties)) {
+			$context = $this->spreadsheetImport->getContext();
+			$contextArguments = $this->settings[$context]['arguments'];
 			if (is_array($contextArguments)) {
-			$arguments = $this->spreadsheetImport->getArguments();
 				foreach ($contextArguments as $contextArgument) {
+					if (isset($contextArgument['identifier']) && $contextArgument['identifier'] == TRUE) {
 						$name = $contextArgument['name'];
+						$arguments = $this->spreadsheetImport->getArguments();
 						if (array_key_exists($name, $arguments)) {
-					if (array_key_exists('domain', $contextArgument)) {
-						$value = $this->propertyMapper->convert($arguments[$name], $contextArgument['domain']);
-					} else {
-						$value = $arguments[$name];
+							$this->argumentIdentifierProperties[$name] = $arguments[$name];
+						}
 					}
-					$setter = 'set' . ucfirst($name);
-					$object->$setter($value);
 				}
 			}
 		}
+		return $this->argumentIdentifierProperties;
+	}
+
+	/**
+	 * @return \TYPO3\Flow\Persistence\RepositoryInterface
+	 */
+	private function getDomainRepository() {
+		$repositoryClassName = preg_replace(array('/\\\Model\\\/', '/$/'), array('\\Repository\\', 'Repository'), $this->domain);
+		/** @var RepositoryInterface $repository */
+		$repository = $this->objectManager->get($repositoryClassName);
+		return $repository;
 	}
 }

Configuration/Settings.yaml

 WE:
   SpreadsheetImport:
-    cleanupImportsThreasholdDays: 365
+    keepPastImportsThreasholdDays: 365
+    maxExecutionThreasholdMinutes: 720
     persistRecordsChunkSize: 100

Configuration/Settings.yaml.example

 WE:
   SpreadsheetImport:
-    cleanupImportsThreasholdDays: 365
+    keepPastImportsThreasholdDays: 365
+    maxExecutionThreasholdMinutes: 720
     persistRecordsChunkSize: 100
     default:
       domain: WE\Sample\Domain\Model\User

Configuration/Testing/Settings.yaml

 WE:
   SpreadsheetImport:
     testing:
-      domain: WE\SpreadsheetImport\Tests\Functional\Fixtures\ImportTarget
-      arguments: ~
+      domain: WE\SpreadsheetImport\Tests\Functional\Fixtures\Domain\Model\ImportTarget
+      arguments:
+        - { name: category, domain: 'WE\SpreadsheetImport\Tests\Functional\Fixtures\Domain\Model\ImportTargetCategory', identifier: TRUE }
+        - { name: comment, static: 'Sample import' }

Migrations/Mysql/Version20161103191621.php

+<?php
+namespace TYPO3\Flow\Persistence\Doctrine\Migrations;
+
+use Doctrine\DBAL\Migrations\AbstractMigration;
+use Doctrine\DBAL\Schema\Schema;
+
+/**
+ * Add progressdate property
+ */
+class Version20161103191621 extends AbstractMigration
+{
+
+    /**
+     * @return string
+     */
+    public function getDescription() {
+        return '';
+    }
+
+    /**
+     * @param Schema $schema
+     * @return void
+     */
+    public function up(Schema $schema)
+    {
+        $this->abortIf($this->connection->getDatabasePlatform()->getName() != 'mysql', 'Migration can only be executed safely on "mysql".');
+
+        $this->addSql('ALTER TABLE we_spreadsheetimport_domain_model_spreadsheetimport ADD progressdate DATETIME NOT NULL');
+    }
+
+    /**
+     * @param Schema $schema
+     * @return void
+     */
+    public function down(Schema $schema)
+    {
+        $this->abortIf($this->connection->getDatabasePlatform()->getName() != 'mysql', 'Migration can only be executed safely on "mysql".');
+
+        $this->addSql('ALTER TABLE we_spreadsheetimport_domain_model_spreadsheetimport DROP progressdate');
+    }
+}

Resources/Private/Templates/SpreadsheetImport/Preview.html

@@ -20,7 +20,7 @@
 		<f:form.hidden name="spreadsheetImport" value="{spreadsheetImport}"/>
 		<f:for each="{preview}" key="property" as="previewMapping">
 			<div>
-				<label>
+				<label class="{f:if(condition: previewMapping.error, then: 'error-text')}">
 					<f:if condition="{previewMapping.mapping.labelId}">
 						<f:then><f:translate id="{previewMapping.mapping.labelId}" />:</f:then>
 						<f:else>{property}:</f:else>

Resources/Private/Translations/de/Main.xlf

@@ -41,7 +41,7 @@
 				<source>Import Vorschau</source>
 			</trans-unit>
 			<trans-unit id="label.spreadsheet_import.preview.invalid_record">
-				<source>Ungültige Daten. Dieser Record wird nicht übersprungen.</source>
+				<source>Ungültige Daten. Dieser Record wird übersprungen.</source>
 			</trans-unit>
 			<trans-unit id="label.spreadsheet_import.preview.record">
 				<source>Zeile</source>

Tests/Functional/Fixtures/Domain/Model/ImportTarget.php

+<?php
+namespace WE\SpreadsheetImport\Tests\Functional\Fixtures\Domain\Model;
+
+/*                                                                        *
+ * This script belongs to the Flow package "SpreadsheetImport".           *
+ *                                                                        *
+ * It is free software; you can redistribute it and/or modify it under    *
+ * the terms of the GNU Lesser General Public License, either version 3   *
+ * of the License, or (at your option) any later version.                 *
+ *                                                                        *
+ * The TYPO3 project - inspiring people to share!                         *
+ *                                                                        */
+
+use TYPO3\Flow\Annotations as Flow;
+use WE\SpreadsheetImport\Annotations as SpreadsheetImport;
+
+/**
+ * @Flow\Entity
+ */
+class ImportTarget {
+
+	/**
+	 * @var string
+	 * @SpreadsheetImport\Mapping(identifier=true, setter="setRawId")
+	 */
+	protected  $id;
+
+	/**
+	 * @var string
+	 * @SpreadsheetImport\Mapping
+	 */
+	protected  $firstName;
+
+	/**
+	 * @var string
+	 * @SpreadsheetImport\Mapping
+	 */
+	protected  $lastName;
+
+	/**
+	 * @var string
+	 * @SpreadsheetImport\Mapping
+	 */
+	protected  $account;
+
+	/**
+	 * @var ImportTargetCategory
+	 */
+	protected  $category;
+
+	/**
+	 * @var string
+	 */
+	protected  $comment;
+
+	/**
+	 * @return string
+	 */
+	public function getId() {
+		return $this->id;
+	}
+
+	/**
+	 * @param string $id
+	 */
+	public function setId($id) {
+		$this->id = $id;
+	}
+
+	/**
+	 * @param string $id
+	 */
+	public function setRawId($id) {
+		$this->id = sprintf('%05d', $id);
+	}
+
+	/**
+	 * @return string
+	 */
+	public function getFirstName() {
+		return $this->firstName;
+	}
+
+	/**
+	 * @param string $firstName
+	 */
+	public function setFirstName($firstName) {
+		$this->firstName = $firstName;
+	}
+
+	/**
+	 * @return string
+	 */
+	public function getLastName() {
+		return $this->lastName;
+	}
+
+	/**
+	 * @param string $lastName
+	 */
+	public function setLastName($lastName) {
+		$this->lastName = $lastName;
+	}
+
+	/**
+	 * @return string
+	 */
+	public function getAccount() {
+		return $this->account;
+	}
+
+	/**
+	 * @param string $account
+	 */
+	public function setAccount($account) {
+		$this->account = $account;
+	}
+
+	/**
+	 * @return \WE\SpreadsheetImport\Tests\Functional\Fixtures\Domain\Model\ImportTargetCategory
+	 */
+	public function getCategory() {
+		return $this->category;
+	}
+
+	/**
+	 * @param \WE\SpreadsheetImport\Tests\Functional\Fixtures\Domain\Model\ImportTargetCategory $category
+	 */
+	public function setCategory($category) {
+		$this->category = $category;
+	}
+
+	/**
+	 * @return string
+	 */
+	public function getComment() {
+		return $this->comment;
+	}
+
+	/**
+	 * @param string $comment
+	 */
+	public function setComment($comment) {
+		$this->comment = $comment;
+	}
+
+}

Tests/Functional/Fixtures/ImportTarget.php → Tests/Functional/Fixtures/Domain/Model/ImportTargetCategory.php

 <?php
-namespace WE\SpreadsheetImport\Tests\Functional\Fixtures;
+namespace WE\SpreadsheetImport\Tests\Functional\Fixtures\Domain\Model;
 
 /*                                                                        *
  * This script belongs to the Flow package "SpreadsheetImport".           *
@@ -13,22 +13,19 @@ namespace WE\SpreadsheetImport\Tests\Functional\Fixtures;
 
 use TYPO3\Flow\Annotations as Flow;
 use WE\SpreadsheetImport\Annotations as SpreadsheetImport;
-use WE\SpreadsheetImport\Domain\Model\SpreadsheetImportTargetInterface;
 
 /**
  * @Flow\Entity
  */
-class ImportTarget {
+class ImportTargetCategory {
 
 	/**
 	 * @var string
-	 * @SpreadsheetImport\Mapping(identifier=true, setter="setRawId")
 	 */
 	protected  $id;
 
 	/**
 	 * @var string
-	 * @SpreadsheetImport\Mapping
 	 */
 	protected  $name;
 
@@ -46,13 +43,6 @@ class ImportTarget {
 		$this->id = $id;
 	}
 
-	/**
-	 * @param string $id
-	 */
-	public function setRawId($id) {
-		$this->id = sprintf('%05d', $id);
-	}
-
 	/**
 	 * @return string
 	 */
@@ -66,4 +56,5 @@ class ImportTarget {
 	public function setName($name) {
 		$this->name = $name;
 	}
+
 }

Tests/Functional/Fixtures/Domain/Repository/ImportTargetRepository.php

+<?php
+namespace WE\SpreadsheetImport\Tests\Functional\Fixtures\Domain\Repository;
+
+/*                                                                        *
+ * This script belongs to the Flow package "SpreadsheetImport".           *
+ *                                                                        *
+ * It is free software; you can redistribute it and/or modify it under    *
+ * the terms of the GNU Lesser General Public License, either version 3   *
+ * of the License, or (at your option) any later version.                 *
+ *                                                                        *
+ * The TYPO3 project - inspiring people to share!                         *
+ *                                                                        */
+
+use TYPO3\Flow\Annotations as Flow;
+use TYPO3\Flow\Persistence\Repository;
+
+/**
+ * @Flow\Scope("singleton")
+ */
+class ImportTargetRepository extends Repository {
+}

Tests/Functional/SpreadsheetImportServiceTest.php

@@ -17,7 +17,8 @@ use TYPO3\Flow\Tests\FunctionalTestCase;
 use WE\SpreadsheetImport\Annotations\Mapping;
 use WE\SpreadsheetImport\Domain\Model\SpreadsheetImport;
 use WE\SpreadsheetImport\SpreadsheetImportService;
-use WE\SpreadsheetImport\Tests\Functional\Fixtures\ImportTarget;
+use WE\SpreadsheetImport\Tests\Functional\Fixtures\Domain\Model\ImportTarget;
+use WE\SpreadsheetImport\Tests\Functional\Fixtures\Domain\Model\ImportTargetCategory;
 
 class SpreadsheetImportServiceTest extends FunctionalTestCase {
 
@@ -41,19 +42,17 @@ class SpreadsheetImportServiceTest extends FunctionalTestCase {
 
 		$reflectionService = $this->objectManager->get(ReflectionService::class);
 		$this->inject($this->spreadsheetImportService, 'reflectionService', $reflectionService);
-
-		$this->initializeSpreadsheetMock();
 	}
 
 	/**
 	 * @return void
 	 */
 	public function tearDown() {
-		$persistenceManager = self::$bootstrap->getObjectManager()->get('TYPO3\Flow\Persistence\PersistenceManagerInterface');
+		$persistenceManager = $this->objectManager->get('TYPO3\Flow\Persistence\PersistenceManagerInterface');
 		if (is_callable(array($persistenceManager, 'tearDown'))) {
 			$persistenceManager->tearDown();
 		}
-		self::$bootstrap->getObjectManager()->forgetInstance('TYPO3\Flow\Persistence\PersistenceManagerInterface');
+		$this->objectManager->forgetInstance('TYPO3\Flow\Persistence\PersistenceManagerInterface');
 		parent::tearDown();
 	}
 
@@ -62,47 +61,75 @@ class SpreadsheetImportServiceTest extends FunctionalTestCase {
 		$spreadsheetImport->setContext('testing');
 		$resource = $this->resourceManager->importResource(__DIR__ . '/Fixtures/Resources/sample.xlsx');
 		$spreadsheetImport->setFile($resource);
-		$idMapping = array('column' => 'C', 'mapping' => new Mapping());
-		$nameMapping = array('column' => 'A', 'mapping' => new Mapping());
-		$spreadsheetImport->setMapping(array('id' => $idMapping, 'name' => $nameMapping));
 		$this->spreadsheetImportService->init($spreadsheetImport);
+
+		return $spreadsheetImport;
+	}
+
+	private function initializeConfiguredSpreadsheetMock() {
+		$spreadsheetImport = $this->initializeSpreadsheetMock();
+		$annotationMappings = $this->spreadsheetImportService->getAnnotationMappingProperties();
+		$spreadsheetImport->setMapping(array(
+			'id' => array('column' => 'C', 'mapping' => $annotationMappings['id']),
+			'firstName' => array('column' => 'A', 'mapping' => $annotationMappings['firstName']),
+			'lastName' => array('column' => 'B', 'mapping' => $annotationMappings['lastName']),
+			'account' => array('column' => 'C', 'mapping' => $annotationMappings['account'])));
+		$spreadsheetImport->setArguments(array(
+			'category' => new ImportTargetCategory(), // Could also simply assign the UUID
+			'comment' => 'Sample import'
+		));
 	}
 
 	/**
 	 * @test
 	 */
-	public function getMappingPropertiesReturnsPropertiesWithMappingAnnotation() {
+	public function getAnnotationMappingPropertiesReturnsArrayMappingAnnotation() {
+		$this->initializeSpreadsheetMock();
 		$properties = $this->spreadsheetImportService->getAnnotationMappingProperties();
 		$this->assertArrayHasKey('id', $properties);
-		$this->assertArrayHasKey('name', $properties);
+		$this->assertArrayHasKey('firstName', $properties);
 		/** @var Mapping $id */
 		$id = $properties['id'];
-		/** @var Mapping $name */
-		$name = $properties['name'];
+		/** @var Mapping $firstName */
+		$firstName = $properties['firstName'];
 		$this->assertSame(TRUE, $id->identifier);
-		$this->assertSame(FALSE, $name->identifier);
+		$this->assertSame(FALSE, $firstName->identifier);
+	}
+
+	/**
+	 * @test
+	 */
+	public function getTotalRecordsRecordsNumberOfObjectsToImport() {
+		$this->initializeSpreadsheetMock();
+		$this->assertSame(2, $this->spreadsheetImportService->getTotalRecords());
 	}
 
 	/**
 	 * @test
 	 */
 	public function getSpreadsheetColumnsReturnsColumnsWithHeadings() {
+		$this->initializeSpreadsheetMock();
 		$columns = $this->spreadsheetImportService->getSpreadsheetColumns();
 		$this->assertArrayHasKey('A', $columns);
 		$this->assertArrayHasKey('B', $columns);
 		$this->assertArrayHasKey('C', $columns);
-		$this->assertContains('name', $columns);
-		$this->assertContains('lastname', $columns);
-		$this->assertContains('id', $columns);
+		$this->assertSame('firstname', $columns['A']);
+		$this->assertSame('name', $columns['B']);
+		$this->assertSame('id', $columns['C']);
 	}
 
 	/**
 	 * @test
 	 */
 	public function getObjectByRowOneReturnsImportTargetWithSetProperties() {
+		$this->initializeConfiguredSpreadsheetMock();
 		/** @var ImportTarget $object */
 		$object = $this->spreadsheetImportService->getObjectByRow(1);
-		$this->assertEquals('00001', $object->getId());
-		$this->assertEquals('Hans', $object->getName());
+		$this->assertSame('00001', $object->getId());
+		$this->assertSame('Hans', $object->getFirstName());
+		$this->assertSame('Muster', $object->getLastName());
+		$this->assertSame('001', $object->getAccount());
+		$this->assertInstanceOf(ImportTargetCategory::class, $object->getCategory());
+		$this->assertSame('Sample import', $object->getComment());
 	}
 }