Merge branch 'KIME-3603' into 'master'

[TASK] Correct importing queue order

refs KIME-3603

See merge request !3

Classes/WE/SpreadsheetImport/Command/SpreadsheetImportCommandController.php

@@ -54,7 +54,7 @@ class SpreadsheetImportCommandController extends CommandController {
 			$this->quit();
 		}
 		/** @var SpreadsheetImport $spreadsheetImport */
-		$spreadsheetImport = $this->spreadsheetImportRepository->findOneByImportingStatus(SpreadsheetImport::IMPORTING_STATUS_IN_QUEUE);
+		$spreadsheetImport = $this->spreadsheetImportRepository->findPreviousOneInQueue();
 		if ($spreadsheetImport instanceof SpreadsheetImport) {
 			// mark importing status as "Progressing" before continuing the importing
 			$spreadsheetImport->setImportingStatus(SpreadsheetImport::IMPORTING_STATUS_IN_PROGRESS);

Classes/WE/SpreadsheetImport/Domain/Repository/SpreadsheetImportRepository.php

@@ -12,7 +12,9 @@ namespace WE\SpreadsheetImport\Domain\Repository;
  *                                                                        */
 
 use TYPO3\Flow\Annotations as Flow;
+use TYPO3\Flow\Persistence\QueryInterface;
 use TYPO3\Flow\Persistence\Repository;
+use WE\SpreadsheetImport\Domain\Model\SpreadsheetImport;
 
 /**
  * @Flow\Scope("singleton")
@@ -22,7 +24,21 @@ class SpreadsheetImportRepository extends Repository {
 	/**
 	 * @var array
 	 */
-	protected $defaultOrderings = array('scheduleDate' => \TYPO3\Flow\Persistence\QueryInterface::ORDER_ASCENDING);
+	protected $defaultOrderings = array('scheduleDate' => QueryInterface::ORDER_DESCENDING);
+
+	/**
+	 * @return SpreadsheetImport
+	 */
+	public function findPreviousOneInQueue() {
+		$query = $this->createQuery();
+		$constraint = $query->logicalAnd(
+			$query->equals('importingStatus', SpreadsheetImport::IMPORTING_STATUS_IN_QUEUE),
+			$query->lessThanOrEqual('scheduleDate', new \DateTime())
+		);
+		return $query->matching($constraint)
+			->setOrderings(array('scheduleDate' => QueryInterface::ORDER_ASCENDING))
+			->execute()->getFirst();
+	}
 
 	/**
 	 * @param \DateTime $dateTime

Classes/WE/SpreadsheetImport/SpreadsheetImportService.php

@@ -190,6 +190,8 @@ class SpreadsheetImportService {
 		$identifierProperties = $this->getDomainMappingIdentifierProperties();
 		$file = $this->spreadsheetImport->getFile()->createTemporaryLocalCopy();
 		$reader = \PHPExcel_IOFactory::load($file);
+		$numberOfRecordsToPersist = $this->settings['numberOfRecordsToPersist'];
+		$i = 0;
 		/** @var \PHPExcel_Worksheet_Row $row */
 		foreach ($reader->getActiveSheet()->getRowIterator(2) as $row) {
 			$object = $this->findObjectByIdentifierPropertiesPerRow($identifierProperties, $row);
@@ -199,6 +201,7 @@ class SpreadsheetImportService {
 					$this->setObjectPropertiesByRow($object, $row);
 					$objectRepository->update($object);
 					$totalUpdated++;
+					$i++;
 				} else {
 					$totalSkipped++;
 				}
@@ -208,17 +211,28 @@ class SpreadsheetImportService {
 				$objectRepository->add($newObject);
 				$objectIds[] = $this->persistenceManager->getIdentifierByObject($newObject);
 				$totalInserted++;
+				$i++;
 			} else {
 				$totalSkipped++;
 			}
+			if ($i >= $numberOfRecordsToPersist) {
+				$this->persistenceManager->persistAll();
+				$i = 0;
+			}
 		}
 
 		// remove objects which are not exist on the spreadsheet
 		if ($this->spreadsheetImport->isDeleting()) {
 			$notExistingObjects = $this->findObjectsByExcludedIds($objectIds);
+			$i = 0;
 			foreach ($notExistingObjects as $object) {
 				$objectRepository->remove($object);
 				$totalDeleted++;
+				$i++;
+				if ($i >= $numberOfRecordsToPersist) {
+					$this->persistenceManager->persistAll();
+					$i = 0;
+				}
 			}
 		}
 

Configuration/Settings.yaml

 WE:
   SpreadsheetImport:
     cleanupImportFromPreviousDay: 5
+    numberOfRecordsToPersist: 100
     swisscomGrb:
         domain: WE\KIME\Domain\Model\User
         Configuration:

Migrations/Mysql/Version20161026173055.php

@@ -26,7 +26,6 @@ class Version20161026173055 extends AbstractMigration
         $this->abortIf($this->connection->getDatabasePlatform()->getName() != 'mysql', 'Migration can only be executed safely on "mysql".');
 
         $this->addSql('ALTER TABLE we_spreadsheetimport_domain_model_spreadsheetimport ADD arguments LONGTEXT NOT NULL');
-        $this->addSql('ALTER TABLE we_spreadsheetimport_domain_model_spreadsheetimport ADD CONSTRAINT FK_19518FA38C9F3610 FOREIGN KEY (file) REFERENCES typo3_flow_resource_resource (persistence_object_identifier)');
     }
 
     /**
@@ -37,7 +36,6 @@ class Version20161026173055 extends AbstractMigration
     {
         $this->abortIf($this->connection->getDatabasePlatform()->getName() != 'mysql', 'Migration can only be executed safely on "mysql".');
 
-        $this->addSql('ALTER TABLE we_spreadsheetimport_domain_model_spreadsheetimport DROP FOREIGN KEY FK_19518FA38C9F3610');
         $this->addSql('ALTER TABLE we_spreadsheetimport_domain_model_spreadsheetimport DROP arguments');
     }
 }