Commit 886ad68d authored by Chivy Lim's avatar Chivy Lim
Browse files

[TASK] Force DB persist by specific amount of records 

refs KIME-3603
parent c6ecbfd1

Classes/WE/SpreadsheetImport/SpreadsheetImportService.php

+14 −0
Original line number Original line Diff line number Diff line
@@ -190,6 +190,8 @@ class SpreadsheetImportService { 
		$identifierProperties = $this->getDomainMappingIdentifierProperties();

		$identifierProperties = $this->getDomainMappingIdentifierProperties();

		$file = $this->spreadsheetImport->getFile()->createTemporaryLocalCopy();

		$file = $this->spreadsheetImport->getFile()->createTemporaryLocalCopy();

		$reader = \PHPExcel_IOFactory::load($file);

		$reader = \PHPExcel_IOFactory::load($file);

		$numberOfRecordsToPersist = $this->settings['numberOfRecordsToPersist'];

		$i = 0;

		/** @var \PHPExcel_Worksheet_Row $row */

		/** @var \PHPExcel_Worksheet_Row $row */

		foreach ($reader->getActiveSheet()->getRowIterator(2) as $row) {

		foreach ($reader->getActiveSheet()->getRowIterator(2) as $row) {

			$object = $this->findObjectByIdentifierPropertiesPerRow($identifierProperties, $row);

			$object = $this->findObjectByIdentifierPropertiesPerRow($identifierProperties, $row);
@@ -199,6 +201,7 @@ class SpreadsheetImportService { 
					$this->setObjectPropertiesByRow($object, $row);

					$this->setObjectPropertiesByRow($object, $row);

					$objectRepository->update($object);

					$objectRepository->update($object);

					$totalUpdated++;

					$totalUpdated++;

					$i++;

				} else {

				} else {

					$totalSkipped++;

					$totalSkipped++;

				}

				}
@@ -208,17 +211,28 @@ class SpreadsheetImportService { 
				$objectRepository->add($newObject);

				$objectRepository->add($newObject);

				$objectIds[] = $this->persistenceManager->getIdentifierByObject($newObject);

				$objectIds[] = $this->persistenceManager->getIdentifierByObject($newObject);

				$totalInserted++;

				$totalInserted++;

				$i++;

			} else {

			} else {

				$totalSkipped++;

				$totalSkipped++;

			}

			}

			if ($i >= $numberOfRecordsToPersist) {

				$this->persistenceManager->persistAll();

				$i = 0;

			}

		}

		}





		// remove objects which are not exist on the spreadsheet

		// remove objects which are not exist on the spreadsheet

		if ($this->spreadsheetImport->isDeleting()) {

		if ($this->spreadsheetImport->isDeleting()) {

			$notExistingObjects = $this->findObjectsByExcludedIds($objectIds);

			$notExistingObjects = $this->findObjectsByExcludedIds($objectIds);

			$i = 0;

			foreach ($notExistingObjects as $object) {

			foreach ($notExistingObjects as $object) {

				$objectRepository->remove($object);

				$objectRepository->remove($object);

				$totalDeleted++;

				$totalDeleted++;

				$i++;

				if ($i >= $numberOfRecordsToPersist) {

					$this->persistenceManager->persistAll();

					$i = 0;

				}

			}

			}

		}

		}

Configuration/Settings.yaml

+1 −0
Original line number Original line Diff line number Diff line 
WE:

WE:

  SpreadsheetImport:

  SpreadsheetImport:

    cleanupImportFromPreviousDay: 5

    cleanupImportFromPreviousDay: 5

    numberOfRecordsToPersist: 100

    swisscomGrb:

    swisscomGrb:

        domain: WE\KIME\Domain\Model\User

        domain: WE\KIME\Domain\Model\User

        Configuration:

        Configuration: