[FEATURE] Implement spreadsheet importing command

refs KIME-3603

Classes/WE/SpreadsheetImport/Annotations/Mapping.php

@@ -33,4 +33,12 @@ final class Mapping {
 	 * @var string
 	 */
 	public $setter;
+
+	/**
+	 * Overwrite the default query property name used for identifiers with overwritten getter
+	 *
+	 * @var string
+	 */
+	public $queryPropertyName;
+
 }

Classes/WE/SpreadsheetImport/Command/SpreadsheetImportCommandController.php

+<?php
+namespace WE\SpreadsheetImport\Command;
+
+/*                                                                        *
+ * This script belongs to the Flow package "SpreadsheetImport".           *
+ *                                                                        *
+ * It is free software; you can redistribute it and/or modify it under    *
+ * the terms of the GNU Lesser General Public License, either version 3   *
+ * of the License, or (at your option) any later version.                 *
+ *                                                                        *
+ * The TYPO3 project - inspiring people to share!                         *
+ *                                                                        */
+
+use TYPO3\Flow\Annotations as Flow;
+use TYPO3\Flow\Cli\CommandController;
+use WE\SpreadsheetImport\Domain\Model\SpreadsheetImport;
+
+/**
+ * @Flow\Scope("singleton")
+ */
+class SpreadsheetImportCommandController extends CommandController {
+
+	/**
+	 * @Flow\Inject
+	 * @var \WE\SpreadsheetImport\Domain\Repository\SpreadsheetImportRepository
+	 */
+	protected $spreadsheetImportRepository;
+
+	/**
+	 * @Flow\Inject
+	 * @var \WE\SpreadsheetImport\SpreadsheetImportService
+	 */
+	protected $spreadsheetImportService;
+
+	/**
+	 * @Flow\Inject
+	 * @var \TYPO3\Flow\Persistence\PersistenceManagerInterface
+	 */
+	protected $persistenceManager;
+
+	/**
+	 * Import one pending queued spreadsheet into Domain data, and it will import the next one if it is done
+	 */
+	public function importCommand() {
+		$currentImportingCount = $this->spreadsheetImportRepository->countByImportingStatus(SpreadsheetImport::IMPORTING_STATUS_IN_PROGRESS);
+		if ($currentImportingCount > 0) {
+			$this->outputFormatted('There is a progressing importing spreadsheet.');
+			$this->quit();
+		}
+		/** @var SpreadsheetImport $spreadsheetImport */
+		$spreadsheetImport = $this->spreadsheetImportRepository->findOneByImportingStatus(SpreadsheetImport::IMPORTING_STATUS_IN_QUEUE);
+		if ($spreadsheetImport instanceof SpreadsheetImport) {
+			// mark importing status as "Progressing" before continuing the importing
+			$spreadsheetImport->setImportingStatus(SpreadsheetImport::IMPORTING_STATUS_IN_PROGRESS);
+			$this->spreadsheetImportRepository->update($spreadsheetImport);
+			$this->persistenceManager->persistAll();
+
+			// do importing and mark its status as "Completed/Failed"
+			$this->spreadsheetImportService->init($spreadsheetImport);
+			$this->spreadsheetImportService->import();
+
+			// mark importing status as "Completed"
+			$spreadsheetImport->setImportingStatus(SpreadsheetImport::IMPORTING_STATUS_COMPLETED);
+			$this->spreadsheetImportRepository->update($spreadsheetImport);
+
+			$this->outputFormatted('Spreadsheet has been exported. (totalInserted: %d, totalUpdated: %d, totalDeleted: %d, totalSkipped: %d)',
+				array($spreadsheetImport->getTotalInserted(), $spreadsheetImport->getTotalUpdated(), $spreadsheetImport->getTotalDeleted(), $spreadsheetImport->getTotalSkipped()));
+		} else {
+			$this->outputFormatted('There is no spreadsheet importing in queue.');
+		}
+	}
+
+}

Classes/WE/SpreadsheetImport/Domain/Model/SpreadsheetImport.php

@@ -19,6 +19,11 @@ use Doctrine\ORM\Mapping as ORM;
  */
 class SpreadsheetImport {
 
+	const IMPORTING_STATUS_IN_QUEUE = 0;
+	const IMPORTING_STATUS_IN_PROGRESS = 1;
+	const IMPORTING_STATUS_COMPLETED = 2;
+	const IMPORTING_STATUS_FAILED = 3;
+
 	/**
 	 * @var string
 	 * @Flow\Validate(type="NotEmpty")
@@ -38,6 +43,7 @@ class SpreadsheetImport {
 
 	/**
 	 * @var string
+	 * @ORM\Column(type="text")
 	 */
 	protected $mapping = '';
 
@@ -56,6 +62,12 @@ class SpreadsheetImport {
 	 */
 	protected $deleting = FALSE;
 
+	/**
+	 * @var int
+	 * @ORM\Column(options={"default": 0})
+	 */
+	protected $importingStatus = self::IMPORTING_STATUS_IN_QUEUE;
+
 	/**
 	 * @var int
 	 */
@@ -181,6 +193,20 @@ class SpreadsheetImport {
 		$this->deleting = $deleting;
 	}
 
+	/**
+	 * @return int
+	 */
+	public function getImportingStatus() {
+		return $this->importingStatus;
+	}
+
+	/**
+	 * @param int $importingStatus
+	 */
+	public function setImportingStatus($importingStatus) {
+		$this->importingStatus = $importingStatus;
+	}
+
 	/**
 	 * @return int
 	 */

Classes/WE/SpreadsheetImport/Domain/Repository/SpreadsheetImportRepository.php

@@ -19,4 +19,9 @@ use TYPO3\Flow\Persistence\Repository;
  */
 class SpreadsheetImportRepository extends Repository {
 
+	/**
+	 * @var array
+	 */
+	protected $defaultOrderings = array('scheduleDate' => \TYPO3\Flow\Persistence\QueryInterface::ORDER_ASCENDING);
+
 }

Classes/WE/SpreadsheetImport/SpreadsheetImportService.php

@@ -12,6 +12,7 @@ namespace WE\SpreadsheetImport;
  *                                                                        */
 
 use TYPO3\Flow\Annotations as Flow;
+use TYPO3\Flow\Persistence\RepositoryInterface;
 use WE\SpreadsheetImport\Annotations\Mapping;
 use WE\SpreadsheetImport\Domain\Model\SpreadsheetImport;
 
@@ -47,6 +48,18 @@ class SpreadsheetImportService {
 	 */
 	protected $resourceManager;
 
+	/**
+	 * @Flow\Inject
+	 * @var \TYPO3\Flow\Persistence\PersistenceManagerInterface
+	 */
+	protected $persistenceManager;
+
+	/**
+	 * @Flow\Inject
+	 * @var \TYPO3\Flow\Object\ObjectManagerInterface
+	 */
+	protected $objectManager;
+
 	/**
 	 * @param \WE\SpreadsheetImport\Domain\Model\SpreadsheetImport $spreadsheetImport
 	 *
@@ -70,6 +83,22 @@ class SpreadsheetImportService {
 		return $domainMappingProperties;
 	}
 
+	/**
+	 * @return array
+	 */
+	public function getDomainMappingIdentifierProperties() {
+		$domainMappingProperties = array();
+		$properties = $this->reflectionService->getPropertyNamesByAnnotation($this->context['domain'], Mapping::class);
+		foreach ($properties as $property) {
+			/** @var Mapping $mapping */
+			$mapping = $this->reflectionService->getPropertyAnnotation($this->context['domain'], $property, Mapping::class);
+			if ($mapping->identifier) {
+				$domainMappingProperties[$property] = $mapping;
+			}
+		}
+		return $domainMappingProperties;
+	}
+
 	/**
 	 * @return array
 	 */
@@ -109,21 +138,104 @@ class SpreadsheetImportService {
 	}
 
 	/**
-	 * @return array
+	 * @return void
 	 */
 	public function import() {
 		// TODO: This simply creates the objects for now without update or delete
-		$objects = array();
+		$totalInserted = 0;
+		$totalUpdated = 0;
+		$totalDeleted = 0;
+		$totalSkipped = 0;
+		$objectIds = array();
 		$domain = $this->context['domain'];
+		$objectRepository = $this->getDomainRepository();
+		$identifierProperties = $this->getDomainMappingIdentifierProperties();
 		$file = $this->spreadsheetImport->getFile()->createTemporaryLocalCopy();
 		$reader = \PHPExcel_IOFactory::load($file);
 		/** @var \PHPExcel_Worksheet_Row $row */
 		foreach ($reader->getActiveSheet()->getRowIterator(2) as $row) {
-			$newObject = new $domain;
-			$this->setObjectPropertiesByRow($newObject, $row);
-			$objects[] = $newObject;
+			$object = $this->findObjectByIdentifierPropertiesPerRow($identifierProperties, $row);
+			if (is_object($object)) {
+				$objectIds[] = $this->persistenceManager->getIdentifierByObject($object);
+				if ($this->spreadsheetImport->isUpdating()) {
+					$this->setObjectPropertiesByRow($object, $row);
+					$objectRepository->update($object);
+					$totalUpdated++;
+				} else {
+					$totalSkipped++;
+				}
+			} elseif ($this->spreadsheetImport->isInserting()) {
+				$newObject = new $domain;
+				$this->setObjectPropertiesByRow($newObject, $row);
+				$objectRepository->add($newObject);
+				$objectIds[] = $this->persistenceManager->getIdentifierByObject($newObject);
+				$totalInserted++;
+			} else {
+				$totalSkipped++;
+			}
 		}
-		return $objects;
+
+		// remove objects which are not exist on the spreadsheet
+		if ($this->spreadsheetImport->isDeleting()) {
+			$notExistingObjects = $this->findObjectsByExcludedIds($objectIds);
+			foreach ($notExistingObjects as $object) {
+				$objectRepository->remove($object);
+				$totalDeleted++;
+			}
+		}
+
+		$this->spreadsheetImport->setTotalInserted($totalInserted);
+		$this->spreadsheetImport->setTotalUpdated($totalUpdated);
+		$this->spreadsheetImport->setTotalDeleted($totalDeleted);
+		$this->spreadsheetImport->setTotalSkipped($totalSkipped);
+	}
+
+	/**
+	 * @return \TYPO3\Flow\Persistence\RepositoryInterface
+	 */
+	private function getDomainRepository() {
+		$domainClassName = $this->context['domain'];
+		$repositoryClassName = preg_replace(array('/\\\Model\\\/', '/$/'), array('\\Repository\\', 'Repository'), $domainClassName);
+		/** @var RepositoryInterface $repository */
+		$repository = $this->objectManager->get($repositoryClassName);
+		return $repository;
+	}
+
+	/**
+	 * @param array $identifierProperties
+	 * @param \PHPExcel_Worksheet_Row $row
+	 *
+	 * @return null|object
+	 */
+	private function findObjectByIdentifierPropertiesPerRow(array $identifierProperties, \PHPExcel_Worksheet_Row $row) {
+		$query = $this->getDomainRepository()->createQuery();
+		$constraints = array();
+		$spreadsheetImportMapping = array_flip($this->spreadsheetImport->getMapping());
+		/** @var Mapping $mapping */
+		foreach ($identifierProperties as $property => $mapping) {
+			$column = $spreadsheetImportMapping[$property];
+			/** @var \PHPExcel_Worksheet_RowCellIterator $cellIterator */
+			$cellIterator = $row->getCellIterator($column, $column);
+			$value = $cellIterator->current()->getValue();
+			$propertyName = $mapping->queryPropertyName ?: $property;
+			$constraints[] = $query->equals($propertyName, $value);
+		}
+		if (!empty($constraints)) {
+			return $query->matching($query->logicalAnd($constraints))->execute()->getFirst();
+		} else {
+			return NULL;
+		}
+	}
+
+	/**
+	 * @param array $identifiers
+	 *
+	 * @return \TYPO3\Flow\Persistence\QueryResultInterface
+	 */
+	private function findObjectsByExcludedIds(array $identifiers) {
+		$query = $this->getDomainRepository()->createQuery();
+		$constraint = $query->logicalNot($query->in('Persistence_Object_Identifier', $identifiers));
+		return $query->matching($constraint)->execute();
 	}
 
 	/**

Migrations/Mysql/Version20161026060418.php

+<?php
+namespace TYPO3\Flow\Persistence\Doctrine\Migrations;
+
+use Doctrine\DBAL\Migrations\AbstractMigration;
+use Doctrine\DBAL\Schema\Schema;
+
+/**
+ * Spreadsheet importing status
+ */
+class Version20161026060418 extends AbstractMigration
+{
+
+    /**
+     * @return string
+     */
+    public function getDescription() {
+        return '';
+    }
+
+    /**
+     * @param Schema $schema
+     * @return void
+     */
+    public function up(Schema $schema)
+    {
+        $this->abortIf($this->connection->getDatabasePlatform()->getName() != 'mysql', 'Migration can only be executed safely on "mysql".');
+
+        $this->addSql('ALTER TABLE we_spreadsheetimport_domain_model_spreadsheetimport ADD importingstatus INT DEFAULT 0 NOT NULL');
+    }
+
+    /**
+     * @param Schema $schema
+     * @return void
+     */
+    public function down(Schema $schema)
+    {
+        $this->abortIf($this->connection->getDatabasePlatform()->getName() != 'mysql', 'Migration can only be executed safely on "mysql".');
+
+        $this->addSql('ALTER TABLE we_spreadsheetimport_domain_model_spreadsheetimport DROP importingstatus');
+    }
+}