Merge branch 'KIME-4583' into 'master'

Bugfixes and improvements

refs KIME-4583

See merge request !15

Classes/WE/SpreadsheetImport/Command/SpreadsheetImportCommandController.php

@@ -51,7 +51,7 @@ class SpreadsheetImportCommandController extends CommandController {
 	public function importCommand() {
 		$currentImportingCount = $this->spreadsheetImportRepository->countByImportingStatus(SpreadsheetImport::IMPORTING_STATUS_IN_PROGRESS);
 		if ($currentImportingCount > 0) {
-			$this->outputFormatted('Previous spreadsheet import is still in progress.');
+			$this->outputLine('Previous spreadsheet import is still in progress.');
 			$this->quit();
 		}
 		/** @var SpreadsheetImport $spreadsheetImport */
@@ -67,35 +67,65 @@ class SpreadsheetImportCommandController extends CommandController {
 			try {
 				$this->spreadsheetImportService->import();
 				$spreadsheetImport->setImportingStatus(SpreadsheetImport::IMPORTING_STATUS_COMPLETED);
-				$this->outputFormatted('Spreadsheet has been imported. %d inserted, %d updated, %d deleted, %d skipped',
+				$this->outputLine('Spreadsheet has been imported. %d inserted, %d updated, %d deleted, %d skipped',
 					array($spreadsheetImport->getTotalInserted(), $spreadsheetImport->getTotalUpdated(), $spreadsheetImport->getTotalDeleted(), $spreadsheetImport->getTotalSkipped()));
-			} catch (Exception $e) {
+			} catch (\Exception $e) {
 				$spreadsheetImport->setImportingStatus(SpreadsheetImport::IMPORTING_STATUS_FAILED);
-				$this->outputFormatted('Spreadsheet import failed.');
+				$this->outputLine('Spreadsheet import failed.');
 			}
+			try {
 				$this->spreadsheetImportRepository->update($spreadsheetImport);
+			} catch (\Exception $e) {
+				$this->outputLine('Spreadsheet import status update error. It remains in progress until cleanup.');
+			}
 		} else {
 			$this->outputFormatted('No spreadsheet import in queue.');
 		}
 	}
 
 	/**
-	 * Cleanup previous spreadsheet imports. Threashold defined in settings.
+	 * Cleanup past and stalled imports.
+	 *
+	 * @param int $keepPastImportsThreasholdDays Overwrites the setting value
+	 * @param int $maxExecutionThreasholdMinutes Overwrites the setting value
 	 */
-	public function cleanupCommand() {
-		$cleanupImportsThreasholdDays = intval($this->settings['cleanupImportsThreasholdDays']);
-		$cleanupFromDate = new \DateTime();
-		$cleanupFromDate->sub(new \DateInterval('P' . $cleanupImportsThreasholdDays . 'D'));
-		$oldSpreadsheetImports = $this->spreadsheetImportRepository->findPreviousImportsBySpecificDate($cleanupFromDate);
-		if ($oldSpreadsheetImports->count() > 0) {
-			/** @var SpreadsheetImport $oldSpreadsheetImport */
-			foreach ($oldSpreadsheetImports as $oldSpreadsheetImport) {
-				$this->spreadsheetImportRepository->remove($oldSpreadsheetImport);
+	public function cleanupCommand($keepPastImportsThreasholdDays = -1, $maxExecutionThreasholdMinutes = -1) {
+		$keepPastImportsThreasholdDays = ($keepPastImportsThreasholdDays >= 0) ? $keepPastImportsThreasholdDays : intval($this->settings['keepPastImportsThreasholdDays']);
+		$maxExecutionThreasholdMinutes = ($maxExecutionThreasholdMinutes >= 0) ? $maxExecutionThreasholdMinutes : intval($this->settings['maxExecutionThreasholdMinutes']);
+		$this->cleanupPastImports($keepPastImportsThreasholdDays);
+		$this->cleanupStalledImports($maxExecutionThreasholdMinutes);
 	}
-			$this->outputLine('%d spreadsheet imports were removed.', array($oldSpreadsheetImports->count()));
-		} else {
-			$this->outputLine('There is no spreadsheet import in queue to remove.');
+
+	/**
+	 * Delete past imports
+	 *
+	 * @param int $keepPastImportsThreasholdDays
+	 */
+	private function cleanupPastImports($keepPastImportsThreasholdDays) {
+		$cleanupFromDateTime = new \DateTime();
+		$cleanupFromDateTime->sub(new \DateInterval('P' . $keepPastImportsThreasholdDays . 'D'));
+		$spreadsheetImports = $this->spreadsheetImportRepository->findBySpecificDateTimeAndImportingStatus($cleanupFromDateTime);
+		/** @var SpreadsheetImport $spreadsheetImport */
+		foreach ($spreadsheetImports as $spreadsheetImport) {
+			$this->spreadsheetImportRepository->remove($spreadsheetImport);
 		}
+		$this->outputLine('%d spreadsheet imports removed.', array($spreadsheetImports->count()));
 	}
 
+	/**
+	 * Set stalled imports to failed
+	 *
+	 * @param int $maxExecutionThreasholdMinutes
+	 */
+	private function cleanupStalledImports($maxExecutionThreasholdMinutes) {
+		$cleanupFromDateTime = new \DateTime();
+		$cleanupFromDateTime->sub(new \DateInterval('PT' . $maxExecutionThreasholdMinutes . 'M'));
+		$spreadsheetImports = $this->spreadsheetImportRepository->findBySpecificDateTimeAndImportingStatus($cleanupFromDateTime, SpreadsheetImport::IMPORTING_STATUS_IN_PROGRESS);
+		/** @var SpreadsheetImport $spreadsheetImport */
+		foreach ($spreadsheetImports as $spreadsheetImport) {
+			$spreadsheetImport->setImportingStatus(SpreadsheetImport::IMPORTING_STATUS_FAILED);
+			$this->spreadsheetImportRepository->update($spreadsheetImport);
+		}
+		$this->outputLine('%d spreadsheet imports set to failed.', array($spreadsheetImports->count()));
+	}
 }

Classes/WE/SpreadsheetImport/Domain/Model/SpreadsheetImport.php

@@ -38,10 +38,19 @@ class SpreadsheetImport {
 	protected $file;
 
 	/**
+	 * DateTime when import is scheduled to progress
+	 *
 	 * @var \DateTime
 	 */
 	protected $scheduleDate;
 
+	/**
+	 * Actual DateTime when import is set to in progress
+	 *
+	 * @var \DateTime
+	 */
+	protected $progressDate;
+
 	/**
 	 * @var string
 	 * @ORM\Column(type="text")
@@ -100,6 +109,7 @@ class SpreadsheetImport {
 	 */
 	public function __construct() {
 		$this->scheduleDate = new \DateTime();
+		$this->progressDate = new \DateTime();
 	}
 
 	/**
@@ -142,6 +152,7 @@ class SpreadsheetImport {
 	 */
 	public function setScheduleDate($scheduleDate) {
 		$this->scheduleDate = $scheduleDate;
+		$this->progressDate = $scheduleDate;
 	}
 
 	/**
@@ -166,7 +177,11 @@ class SpreadsheetImport {
 	 * @return array
 	 */
 	public function getArguments() {
-		return unserialize($this->arguments);
+		$arguments = unserialize($this->arguments);
+		if (! is_array($arguments)) {
+			$arguments = array();
+		}
+		return $arguments;
 	}
 
 	/**
@@ -234,6 +249,9 @@ class SpreadsheetImport {
 	 */
 	public function setImportingStatus($importingStatus) {
 		$this->importingStatus = $importingStatus;
+		if ($importingStatus === self::IMPORTING_STATUS_IN_PROGRESS) {
+			$this->progressDate = new \DateTime();
+		}
 	}
 
 	/**

Classes/WE/SpreadsheetImport/Domain/Repository/SpreadsheetImportRepository.php

@@ -42,12 +42,16 @@ class SpreadsheetImportRepository extends Repository {
 
 	/**
 	 * @param \DateTime $dateTime
+	 * @param int $importingStatus
 	 *
 	 * @return \TYPO3\Flow\Persistence\QueryResultInterface
 	 */
-	public function findPreviousImportsBySpecificDate(\DateTime $dateTime) {
+	public function findBySpecificDateTimeAndImportingStatus(\DateTime $dateTime, $importingStatus = -1) {
 		$query = $this->createQuery();
-		$constraint = $query->lessThanOrEqual('scheduleDate', $dateTime);
+		$constraint = $query->lessThanOrEqual('progressDate', $dateTime);
+		if ($importingStatus >= 0) {
+			$constraint = $query->logicalAnd($constraint, $query->equals('importingStatus', $importingStatus));
+		}
 		return $query->matching($constraint)->execute();
 	}
 

Classes/WE/SpreadsheetImport/FrontendMappingService.php

@@ -122,7 +122,7 @@ class FrontendMappingService {
 		$previewObject = $this->spreadsheetImportService->getObjectByRow($record);
 		$preview = array();
 		$hasErrors = FALSE;
-		$objectValidator = $this->validatorResolver->getBaseValidatorConjunction($domain);
+		$objectValidator = $this->validatorResolver->getBaseValidatorConjunction($domain, SpreadsheetImportService::VALIDATION_GROUPS);
 		$errors = $objectValidator->validate($previewObject)->getFlattenedErrors();
 		foreach ($mapping as $property => $columnMapping) {
 			/** @var Mapping $mapping */

Configuration/Settings.yaml

 WE:
   SpreadsheetImport:
-    cleanupImportsThreasholdDays: 365
+    keepPastImportsThreasholdDays: 365
+    maxExecutionThreasholdMinutes: 720
     persistRecordsChunkSize: 100

Configuration/Settings.yaml.example

 WE:
   SpreadsheetImport:
-    cleanupImportsThreasholdDays: 365
+    keepPastImportsThreasholdDays: 365
+    maxExecutionThreasholdMinutes: 720
     persistRecordsChunkSize: 100
     default:
       domain: WE\Sample\Domain\Model\User

Configuration/Testing/Settings.yaml

 WE:
   SpreadsheetImport:
     testing:
-      domain: WE\SpreadsheetImport\Tests\Functional\Fixtures\ImportTarget
-      arguments: ~
+      domain: WE\SpreadsheetImport\Tests\Functional\Fixtures\Domain\Model\ImportTarget
+      arguments:
+        - { name: category, domain: 'WE\SpreadsheetImport\Tests\Functional\Fixtures\Domain\Model\ImportTargetCategory', identifier: TRUE }
+        - { name: comment, static: 'Sample import' }

Migrations/Mysql/Version20161103191621.php

+<?php
+namespace TYPO3\Flow\Persistence\Doctrine\Migrations;
+
+use Doctrine\DBAL\Migrations\AbstractMigration;
+use Doctrine\DBAL\Schema\Schema;
+
+/**
+ * Add progressdate property
+ */
+class Version20161103191621 extends AbstractMigration
+{
+
+    /**
+     * @return string
+     */
+    public function getDescription() {
+        return '';
+    }
+
+    /**
+     * @param Schema $schema
+     * @return void
+     */
+    public function up(Schema $schema)
+    {
+        $this->abortIf($this->connection->getDatabasePlatform()->getName() != 'mysql', 'Migration can only be executed safely on "mysql".');
+
+        $this->addSql('ALTER TABLE we_spreadsheetimport_domain_model_spreadsheetimport ADD progressdate DATETIME NOT NULL');
+    }
+
+    /**
+     * @param Schema $schema
+     * @return void
+     */
+    public function down(Schema $schema)
+    {
+        $this->abortIf($this->connection->getDatabasePlatform()->getName() != 'mysql', 'Migration can only be executed safely on "mysql".');
+
+        $this->addSql('ALTER TABLE we_spreadsheetimport_domain_model_spreadsheetimport DROP progressdate');
+    }
+}

Resources/Private/Templates/SpreadsheetImport/Preview.html

@@ -20,7 +20,7 @@
 		<f:form.hidden name="spreadsheetImport" value="{spreadsheetImport}"/>
 		<f:for each="{preview}" key="property" as="previewMapping">
 			<div>
-				<label>
+				<label class="{f:if(condition: previewMapping.error, then: 'error-text')}">
 					<f:if condition="{previewMapping.mapping.labelId}">
 						<f:then><f:translate id="{previewMapping.mapping.labelId}" />:</f:then>
 						<f:else>{property}:</f:else>

Resources/Private/Translations/de/Main.xlf

@@ -41,7 +41,7 @@
 				<source>Import Vorschau</source>
 			</trans-unit>
 			<trans-unit id="label.spreadsheet_import.preview.invalid_record">
-				<source>Ungültige Daten. Dieser Record wird nicht übersprungen.</source>
+				<source>Ungültige Daten. Dieser Record wird übersprungen.</source>
 			</trans-unit>
 			<trans-unit id="label.spreadsheet_import.preview.record">
 				<source>Zeile</source>

Tests/Functional/Fixtures/Domain/Model/ImportTarget.php

+<?php
+namespace WE\SpreadsheetImport\Tests\Functional\Fixtures\Domain\Model;
+
+/*                                                                        *
+ * This script belongs to the Flow package "SpreadsheetImport".           *
+ *                                                                        *
+ * It is free software; you can redistribute it and/or modify it under    *
+ * the terms of the GNU Lesser General Public License, either version 3   *
+ * of the License, or (at your option) any later version.                 *
+ *                                                                        *
+ * The TYPO3 project - inspiring people to share!                         *
+ *                                                                        */
+
+use TYPO3\Flow\Annotations as Flow;
+use WE\SpreadsheetImport\Annotations as SpreadsheetImport;
+
+/**
+ * @Flow\Entity
+ */
+class ImportTarget {
+
+	/**
+	 * @var string
+	 * @SpreadsheetImport\Mapping(identifier=true, setter="setRawId")
+	 */
+	protected  $id;
+
+	/**
+	 * @var string
+	 * @SpreadsheetImport\Mapping
+	 */
+	protected  $firstName;
+
+	/**
+	 * @var string
+	 * @SpreadsheetImport\Mapping
+	 */
+	protected  $lastName;
+
+	/**
+	 * @var string
+	 * @SpreadsheetImport\Mapping
+	 */
+	protected  $account;
+
+	/**
+	 * @var ImportTargetCategory
+	 */
+	protected  $category;
+
+	/**
+	 * @var string
+	 */
+	protected  $comment;
+
+	/**
+	 * @return string
+	 */
+	public function getId() {
+		return $this->id;
+	}
+
+	/**
+	 * @param string $id
+	 */
+	public function setId($id) {
+		$this->id = $id;
+	}
+
+	/**
+	 * @param string $id
+	 */
+	public function setRawId($id) {
+		$this->id = sprintf('%05d', $id);
+	}
+
+	/**
+	 * @return string
+	 */
+	public function getFirstName() {
+		return $this->firstName;
+	}
+
+	/**
+	 * @param string $firstName
+	 */
+	public function setFirstName($firstName) {
+		$this->firstName = $firstName;
+	}
+
+	/**
+	 * @return string
+	 */
+	public function getLastName() {
+		return $this->lastName;
+	}
+
+	/**
+	 * @param string $lastName
+	 */
+	public function setLastName($lastName) {
+		$this->lastName = $lastName;
+	}
+
+	/**
+	 * @return string
+	 */
+	public function getAccount() {
+		return $this->account;
+	}
+
+	/**
+	 * @param string $account
+	 */
+	public function setAccount($account) {
+		$this->account = $account;
+	}
+
+	/**
+	 * @return \WE\SpreadsheetImport\Tests\Functional\Fixtures\Domain\Model\ImportTargetCategory
+	 */
+	public function getCategory() {
+		return $this->category;
+	}
+
+	/**
+	 * @param \WE\SpreadsheetImport\Tests\Functional\Fixtures\Domain\Model\ImportTargetCategory $category
+	 */
+	public function setCategory($category) {
+		$this->category = $category;
+	}
+
+	/**
+	 * @return string
+	 */
+	public function getComment() {
+		return $this->comment;
+	}
+
+	/**
+	 * @param string $comment
+	 */
+	public function setComment($comment) {
+		$this->comment = $comment;
+	}
+
+}

Tests/Functional/Fixtures/ImportTarget.php → Tests/Functional/Fixtures/Domain/Model/ImportTargetCategory.php

 <?php
-namespace WE\SpreadsheetImport\Tests\Functional\Fixtures;
+namespace WE\SpreadsheetImport\Tests\Functional\Fixtures\Domain\Model;
 
 /*                                                                        *
  * This script belongs to the Flow package "SpreadsheetImport".           *
@@ -13,22 +13,19 @@ namespace WE\SpreadsheetImport\Tests\Functional\Fixtures;
 
 use TYPO3\Flow\Annotations as Flow;
 use WE\SpreadsheetImport\Annotations as SpreadsheetImport;
-use WE\SpreadsheetImport\Domain\Model\SpreadsheetImportTargetInterface;
 
 /**
  * @Flow\Entity
  */
-class ImportTarget {
+class ImportTargetCategory {
 
 	/**
 	 * @var string
-	 * @SpreadsheetImport\Mapping(identifier=true, setter="setRawId")
 	 */
 	protected  $id;
 
 	/**
 	 * @var string
-	 * @SpreadsheetImport\Mapping
 	 */
 	protected  $name;
 
@@ -46,13 +43,6 @@ class ImportTarget {
 		$this->id = $id;
 	}
 
-	/**
-	 * @param string $id
-	 */
-	public function setRawId($id) {
-		$this->id = sprintf('%05d', $id);
-	}
-
 	/**
 	 * @return string
 	 */
@@ -66,4 +56,5 @@ class ImportTarget {
 	public function setName($name) {
 		$this->name = $name;
 	}
+
 }

Tests/Functional/Fixtures/Domain/Repository/ImportTargetRepository.php

+<?php
+namespace WE\SpreadsheetImport\Tests\Functional\Fixtures\Domain\Repository;
+
+/*                                                                        *
+ * This script belongs to the Flow package "SpreadsheetImport".           *
+ *                                                                        *
+ * It is free software; you can redistribute it and/or modify it under    *
+ * the terms of the GNU Lesser General Public License, either version 3   *
+ * of the License, or (at your option) any later version.                 *
+ *                                                                        *
+ * The TYPO3 project - inspiring people to share!                         *
+ *                                                                        */
+
+use TYPO3\Flow\Annotations as Flow;
+use TYPO3\Flow\Persistence\Repository;
+
+/**
+ * @Flow\Scope("singleton")
+ */
+class ImportTargetRepository extends Repository {
+}

Tests/Functional/SpreadsheetImportServiceTest.php

@@ -17,7 +17,8 @@ use TYPO3\Flow\Tests\FunctionalTestCase;
 use WE\SpreadsheetImport\Annotations\Mapping;
 use WE\SpreadsheetImport\Domain\Model\SpreadsheetImport;
 use WE\SpreadsheetImport\SpreadsheetImportService;
-use WE\SpreadsheetImport\Tests\Functional\Fixtures\ImportTarget;
+use WE\SpreadsheetImport\Tests\Functional\Fixtures\Domain\Model\ImportTarget;
+use WE\SpreadsheetImport\Tests\Functional\Fixtures\Domain\Model\ImportTargetCategory;
 
 class SpreadsheetImportServiceTest extends FunctionalTestCase {
 
@@ -41,19 +42,17 @@ class SpreadsheetImportServiceTest extends FunctionalTestCase {
 
 		$reflectionService = $this->objectManager->get(ReflectionService::class);
 		$this->inject($this->spreadsheetImportService, 'reflectionService', $reflectionService);
-
-		$this->initializeSpreadsheetMock();
 	}
 
 	/**
 	 * @return void
 	 */
 	public function tearDown() {
-		$persistenceManager = self::$bootstrap->getObjectManager()->get('TYPO3\Flow\Persistence\PersistenceManagerInterface');
+		$persistenceManager = $this->objectManager->get('TYPO3\Flow\Persistence\PersistenceManagerInterface');
 		if (is_callable(array($persistenceManager, 'tearDown'))) {
 			$persistenceManager->tearDown();
 		}
-		self::$bootstrap->getObjectManager()->forgetInstance('TYPO3\Flow\Persistence\PersistenceManagerInterface');
+		$this->objectManager->forgetInstance('TYPO3\Flow\Persistence\PersistenceManagerInterface');
 		parent::tearDown();
 	}
 
@@ -62,47 +61,75 @@ class SpreadsheetImportServiceTest extends FunctionalTestCase {
 		$spreadsheetImport->setContext('testing');
 		$resource = $this->resourceManager->importResource(__DIR__ . '/Fixtures/Resources/sample.xlsx');
 		$spreadsheetImport->setFile($resource);
-		$idMapping = array('column' => 'C', 'mapping' => new Mapping());
-		$nameMapping = array('column' => 'A', 'mapping' => new Mapping());
-		$spreadsheetImport->setMapping(array('id' => $idMapping, 'name' => $nameMapping));
 		$this->spreadsheetImportService->init($spreadsheetImport);
+
+		return $spreadsheetImport;
+	}
+
+	private function initializeConfiguredSpreadsheetMock() {
+		$spreadsheetImport = $this->initializeSpreadsheetMock();
+		$annotationMappings = $this->spreadsheetImportService->getAnnotationMappingProperties();
+		$spreadsheetImport->setMapping(array(
+			'id' => array('column' => 'C', 'mapping' => $annotationMappings['id']),
+			'firstName' => array('column' => 'A', 'mapping' => $annotationMappings['firstName']),
+			'lastName' => array('column' => 'B', 'mapping' => $annotationMappings['lastName']),
+			'account' => array('column' => 'C', 'mapping' => $annotationMappings['account'])));
+		$spreadsheetImport->setArguments(array(
+			'category' => new ImportTargetCategory(), // Could also simply assign the UUID
+			'comment' => 'Sample import'
+		));
 	}
 
 	/**
 	 * @test
 	 */
-	public function getMappingPropertiesReturnsPropertiesWithMappingAnnotation() {
+	public function getAnnotationMappingPropertiesReturnsArrayMappingAnnotation() {
+		$this->initializeSpreadsheetMock();
 		$properties = $this->spreadsheetImportService->getAnnotationMappingProperties();
 		$this->assertArrayHasKey('id', $properties);
-		$this->assertArrayHasKey('name', $properties);
+		$this->assertArrayHasKey('firstName', $properties);
 		/** @var Mapping $id */
 		$id = $properties['id'];
-		/** @var Mapping $name */
-		$name = $properties['name'];
+		/** @var Mapping $firstName */
+		$firstName = $properties['firstName'];
 		$this->assertSame(TRUE, $id->identifier);
-		$this->assertSame(FALSE, $name->identifier);
+		$this->assertSame(FALSE, $firstName->identifier);
+	}
+
+	/**
+	 * @test
+	 */
+	public function getTotalRecordsRecordsNumberOfObjectsToImport() {
+		$this->initializeSpreadsheetMock();
+		$this->assertSame(2, $this->spreadsheetImportService->getTotalRecords());
 	}
 
 	/**
 	 * @test
 	 */
 	public function getSpreadsheetColumnsReturnsColumnsWithHeadings() {
+		$this->initializeSpreadsheetMock();
 		$columns = $this->spreadsheetImportService->getSpreadsheetColumns();
 		$this->assertArrayHasKey('A', $columns);
 		$this->assertArrayHasKey('B', $columns);
 		$this->assertArrayHasKey('C', $columns);
-		$this->assertContains('name', $columns);
-		$this->assertContains('lastname', $columns);
-		$this->assertContains('id', $columns);
+		$this->assertSame('firstname', $columns['A']);
+		$this->assertSame('name', $columns['B']);
+		$this->assertSame('id', $columns['C']);
 	}
 
 	/**
 	 * @test
 	 */
 	public function getObjectByRowOneReturnsImportTargetWithSetProperties() {
+		$this->initializeConfiguredSpreadsheetMock();
 		/** @var ImportTarget $object */
 		$object = $this->spreadsheetImportService->getObjectByRow(1);
-		$this->assertEquals('00001', $object->getId());
-		$this->assertEquals('Hans', $object->getName());
+		$this->assertSame('00001', $object->getId());
+		$this->assertSame('Hans', $object->getFirstName());
+		$this->assertSame('Muster', $object->getLastName());
+		$this->assertSame('001', $object->getAccount());
+		$this->assertInstanceOf(ImportTargetCategory::class, $object->getCategory());
+		$this->assertSame('Sample import', $object->getComment());
 	}
 }