diff --git a/app/config/service/loader.yml b/app/config/service/loader.yml
index 58a8ef8..9bd585d 100644
--- a/app/config/service/loader.yml
+++ b/app/config/service/loader.yml
@@ -3,11 +3,7 @@ services:
abstract: true
class: AppBundle\Service\Loader\BaseLoader
calls:
- - [setArguments, ['@doctrine.orm.entity_manager', '@monolog.logger.loader']]
-
- service.ldview:
- class: AppBundle\Service\LDViewService
- arguments: ['%ldview_bin%', '@oneup_flysystem.media_filesystem']
+ - [setArguments, ['@doctrine.orm.entity_manager', '@monolog.logger.loader', '@app.transformer.format']]
service.loader.rebrickable:
class: AppBundle\Service\Loader\RebrickableLoader
diff --git a/app/config/service/service.yml b/app/config/service/service.yml
index 0013d14..7eb05a6 100644
--- a/app/config/service/service.yml
+++ b/app/config/service/service.yml
@@ -14,3 +14,7 @@ services:
service.renderer.stl:
class: AppBundle\Service\StlRendererService
arguments: ['%kernel.root_dir%/Resources/povray_layout/layout.tmpl', '%povray_bin%','%stl2pov_bin%']
+
+ service.ldview:
+ class: AppBundle\Service\LDViewService
+ arguments: ['%ldview_bin%', '@oneup_flysystem.media_filesystem']
diff --git a/src/AppBundle/Command/InitDataCommand.php b/src/AppBundle/Command/InitDataCommand.php
new file mode 100644
index 0000000..2d2adc5
--- /dev/null
+++ b/src/AppBundle/Command/InitDataCommand.php
@@ -0,0 +1,67 @@
+setName('app:init')
+ ->setDescription('Loads relations between LDraw models and Rebrickable parts.')
+ ->setHelp('This command allows you to load relation between models and parts into database.')
+ ->setDefinition(
+ new InputDefinition([
+ new InputArgument('ldraw', InputArgument::REQUIRED, 'Path to LDraw library directory'),
+ ])
+ );
+ }
+
+ protected function execute(InputInterface $input, OutputInterface $output)
+ {
+ $loadModelsCommand = $this->getApplication()->find('app:load:models');
+
+ $returnCode = $loadModelsCommand->run(new ArrayInput([
+ 'command' => 'app:load:models',
+ 'ldraw' => $input->getArgument('ldraw'),
+ '--all' => true
+ ]),$output);
+
+ if($returnCode) {
+ return 1;
+ }
+
+ $loadRebrickableCommad = $this->getApplication()->find('app:load:rebrickable');
+ $returnCode = $loadRebrickableCommad->run(new ArrayInput([
+ 'command' => 'app:load:rebrickable']),$output);
+
+ if($returnCode) {
+ return 1;
+ }
+
+ $loadRelationsCommand = $this->getApplication()->find('app:load:relations');
+
+ $returnCode = $loadRelationsCommand->run(new ArrayInput([
+ 'command' => 'app:load:relations']),$output);
+
+ if($returnCode) {
+ return 1;
+ }
+
+ $loadImagesCommand = $this->getApplication()->find('app:load:images');
+
+ $returnCode = $loadImagesCommand->run(new ArrayInput([
+ 'command' => 'app:load:images',
+ '--color' => -1,
+ '--rebrickable' => true,
+ '--models' => true
+ ]),$output);
+ }
+}
diff --git a/src/AppBundle/Command/LoadModelsCommand.php b/src/AppBundle/Command/LoadModelsCommand.php
index c115440..54f925e 100644
--- a/src/AppBundle/Command/LoadModelsCommand.php
+++ b/src/AppBundle/Command/LoadModelsCommand.php
@@ -23,7 +23,6 @@ class LoadModelsCommand extends ContainerAwareCommand
->setDefinition(
new InputDefinition([
new InputArgument('ldraw', InputArgument::REQUIRED, 'Path to LDraw library directory'),
-// new InputOption('images', 'i', InputOption::VALUE_NONE, 'Do you want to generate images of models?'),
new InputOption('all', 'a', InputOption::VALUE_NONE, 'Load all models from LDraw libary folder (/parts directory)'),
new InputOption('file', 'f', InputOption::VALUE_REQUIRED, 'Load single modle into database'),
new InputOption('update', 'u', InputOption::VALUE_NONE, 'Overwrite already loaded models'),
@@ -35,8 +34,7 @@ class LoadModelsCommand extends ContainerAwareCommand
{
if (!$this->lock()) {
$output->writeln('The command is already running in another process.');
-
- return 0;
+ return 1;
}
$modelLoader = $this->getContainer()->get('service.loader.model');
@@ -51,12 +49,10 @@ class LoadModelsCommand extends ContainerAwareCommand
if (($path = $input->getOption('file')) != null) {
if ($file = realpath($path)) {
$output->writeln([
- 'Loading model',
- 'path: '.$file,
- '------------------------------------------------------------------------------',
+ "Loading model: {$path}",
]);
- $modelLoader->loadOneModel($file);
+ $modelLoader->loadOne($file);
$errorCount = $this->getContainer()->get('monolog.logger.loader')->countErrors();
$errors = $errorCount ? ''.$errorCount.'' : '0';
@@ -70,10 +66,12 @@ class LoadModelsCommand extends ContainerAwareCommand
// Load all models inside ldraw/parts directory
if ($input->getOption('all')) {
$output->writeln([
- 'Loading models from LDraw library: '.$ldrawPath.'',
+ '------------------------------------------------------------------------------>',
+ "Loading models from LDraw library:> {$ldrawPath}",
+ '------------------------------------------------------------------------------>',
]);
- $modelLoader->loadAllModels();
+ $modelLoader->loadAll();
$errorCount = $this->getContainer()->get('monolog.logger.loader')->countErrors();
$errors = $errorCount ? ''.$errorCount.'' : '0';
@@ -81,7 +79,9 @@ class LoadModelsCommand extends ContainerAwareCommand
$output->writeln(['Done with "'.$errors.'" errors.']);
}
} else {
- $output->writeln($ldraw.' is not valid path');
+ $output->writeln("{$ldraw} is not a valid path!");
+ $this->release();
+ return 1;
}
$this->release();
diff --git a/src/AppBundle/Command/LoadRebrickableDataCommand.php b/src/AppBundle/Command/LoadRebrickableDataCommand.php
index 71d7135..6ac5cb4 100644
--- a/src/AppBundle/Command/LoadRebrickableDataCommand.php
+++ b/src/AppBundle/Command/LoadRebrickableDataCommand.php
@@ -2,6 +2,7 @@
namespace AppBundle\Command;
+use League\Flysystem\Exception;
use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
@@ -21,7 +22,11 @@ class LoadRebrickableDataCommand extends ContainerAwareCommand
$rebrickableLoader = $this->getContainer()->get('service.loader.rebrickable');
$rebrickableLoader->setOutput($output);
- //TODO log errors
- $rebrickableLoader->loadTables();
+ try {
+ $rebrickableLoader->loadAll();
+ } catch (Exception $exception) {
+ $output->writeln("{$exception->getMessage()}");
+ return 1;
+ }
}
}
diff --git a/src/AppBundle/Command/LoadRelationCommand.php b/src/AppBundle/Command/LoadRelationCommand.php
index d90ebeb..1d18ffe 100644
--- a/src/AppBundle/Command/LoadRelationCommand.php
+++ b/src/AppBundle/Command/LoadRelationCommand.php
@@ -21,7 +21,15 @@ class LoadRelationCommand extends ContainerAwareCommand
$relationLoader = $this->getContainer()->get('service.loader.relation');
$relationLoader->setOutput($output);
- //TODO log errors
- $relationLoader->loadNotPaired();
+
+ $output->writeln([
+ '------------------------------------------------------------------------------>',
+ "Loading relations between parts and models...>",
+ '------------------------------------------------------------------------------>',
+ ]);
+
+ $relationLoader->loadAll();
+
+ $output->writeln(['Done!']);
}
}
diff --git a/src/AppBundle/Service/Loader/BaseLoader.php b/src/AppBundle/Service/Loader/BaseLoader.php
index 07343ee..8e222e2 100644
--- a/src/AppBundle/Service/Loader/BaseLoader.php
+++ b/src/AppBundle/Service/Loader/BaseLoader.php
@@ -4,6 +4,7 @@ namespace AppBundle\Service\Loader;
use AppBundle\Exception\FileNotFoundException;
use AppBundle\Exception\WriteErrorException;
+use AppBundle\Transformer\FormatTransformer;
use Doctrine\ORM\EntityManager;
use Monolog\Logger;
use Symfony\Bundle\FrameworkBundle\Translation\Translator;
@@ -29,21 +30,29 @@ abstract class BaseLoader
*/
protected $progressBar;
- /** @var Logger */
+ /**
+ * @var Logger
+ */
protected $logger;
+ /**
+ * @var FormatTransformer
+ */
+ private $formatTransformer;
+
/**
* Loader constructor.
*
* @param EntityManager $em
* @param Translator $translator
*/
- public function setArguments(EntityManager $em, $logger)
+ public function setArguments(EntityManager $em, $logger, $formatTransformer)
{
$this->em = $em;
$this->em->getConnection()->getConfiguration()->setSQLLogger(null);
$this->logger = $logger;
+ $this->formatTransformer = $formatTransformer;
}
public function setOutput(OutputInterface $output)
@@ -57,28 +66,33 @@ abstract class BaseLoader
*
* @param $total
*/
- protected function initProgressBar($total)
+ protected function initProgressBar($total, $format = 'loader')
{
$this->progressBar = new ProgressBar($this->output, $total);
-// $this->progressBar->setFormat('very_verbose');
- $this->progressBar->setFormat('[%current%/%max%] [%bar%] %percent:3s%% (%elapsed:6s%/%estimated:-6s%) (%filename%)'.PHP_EOL);
+ ProgressBar::setFormatDefinition('loader', '[%current% / %max%] [%bar%] %percent:3s%% (%elapsed:6s%/%estimated:-6s%) (%message%)' . PHP_EOL);
+ ProgressBar::setFormatDefinition('download', '[%progress% / %size%] [%bar%] %percent:3s%% (%elapsed:6s%/%estimated:-6s%)' . PHP_EOL);
+ $this->progressBar->setFormat($format);
$this->progressBar->setBarWidth(70);
+
$this->progressBar->start();
}
protected function progressCallback($notification_code, $severity, $message, $message_code, $bytes_transferred, $bytes_max)
{
+ $this->logger->info($notification_code);
+
switch ($notification_code) {
case STREAM_NOTIFY_FILE_SIZE_IS:
$this->initProgressBar($bytes_max);
- $this->progressBar->setFormat('[%current%/%max%] [%bar%] %percent:3s%% (%elapsed:6s%/%estimated:-6s%)'.PHP_EOL);
+ $this->progressBar->setFormat('download');
+ $this->progressBar->setMessage($this->formatTransformer->bytesToSize($bytes_max),'size');
+ $this->progressBar->setRedrawFrequency($bytes_max/20);
break;
case STREAM_NOTIFY_PROGRESS:
$this->progressBar->setProgress($bytes_transferred);
+ $this->progressBar->setMessage($this->formatTransformer->bytesToSize($bytes_transferred),'progress');
break;
case STREAM_NOTIFY_COMPLETED:
- $this->progressBar->setMessage('Done');
- $this->progressBar->setProgress($bytes_max);
$this->progressBar->finish();
break;
}
@@ -106,6 +120,7 @@ abstract class BaseLoader
if (false === file_put_contents($temp, fopen($url, 'r', 0, $ctx))) {
throw new WriteErrorException($temp);
}
+ $this->progressBar->finish();
} catch (ContextErrorException $e) {
throw new FileNotFoundException($url);
} catch (\Exception $e) {
@@ -114,4 +129,9 @@ abstract class BaseLoader
return $temp;
}
+
+ protected function writeOutput(array $lines) {
+ if($this->output)
+ $this->output->writeln($lines);
+ }
}
diff --git a/src/AppBundle/Service/Loader/ModelLoader.php b/src/AppBundle/Service/Loader/ModelLoader.php
index 6e47507..64f08c8 100644
--- a/src/AppBundle/Service/Loader/ModelLoader.php
+++ b/src/AppBundle/Service/Loader/ModelLoader.php
@@ -12,15 +12,14 @@ use AppBundle\Exception\ConvertingFailedException;
use AppBundle\Exception\FileException;
use AppBundle\Exception\ParseErrorException;
use AppBundle\Service\LDViewService;
-use AppBundle\Utils\LDModelParser;
-use AppBundle\Utils\RelationMapper;
+use AppBundle\Util\LDModelParser;
+use AppBundle\Util\RelationMapper;
use League\Flysystem\Adapter\Local;
use League\Flysystem\Exception;
use League\Flysystem\Filesystem;
use Symfony\Component\Finder\Finder;
use Symfony\Component\Finder\SplFileInfo;
-//TODO refactor
class ModelLoader extends BaseLoader
{
/**
@@ -80,7 +79,7 @@ class ModelLoader extends BaseLoader
}
}
- public function loadOneModel($file)
+ public function loadOne($file)
{
$connection = $this->em->getConnection();
try {
@@ -95,7 +94,7 @@ class ModelLoader extends BaseLoader
}
}
- public function loadAllModels()
+ public function loadAll()
{
$files = $this->finder->in([
$this->ldrawLibraryContext->getAdapter()->getPathPrefix(),
@@ -109,7 +108,7 @@ class ModelLoader extends BaseLoader
$connection->beginTransaction();
try {
- $this->progressBar->setMessage($file->getFilename(), 'filename');
+ $this->progressBar->setMessage($file->getFilename());
$this->loadModel($file->getRealPath());
@@ -127,7 +126,8 @@ class ModelLoader extends BaseLoader
}
/**
- * Load Model entity into database.
+ * Load model entity and all related submodels into database while generating stl file of model.
+ * Uses LDView to convert LDraw .dat to .stl
*
* @param $file
*
@@ -144,7 +144,8 @@ class ModelLoader extends BaseLoader
// Parse model file save data to $modelArray
try {
- $modelArray = $this->ldModelParser->parse($file);
+ $content = file_get_contents($file);
+ $modelArray = $this->ldModelParser->parse($content);
} catch (ParseErrorException $e) {
$this->logger->error($e->getMessage(), [$file]);
@@ -172,8 +173,8 @@ class ModelLoader extends BaseLoader
return $parentModel;
}
- // Load model
+ // Load model
$model = $this->em->getRepository(Model::class)->getOrCreate($modelArray['id']);
// Recursively load models of subparts
@@ -203,7 +204,7 @@ class ModelLoader extends BaseLoader
try {
// update model only if newer version
if (!$model->getModified() || ($model->getModified() < $modelArray['modified'])) {
- $stl = $this->LDViewService->datToStl($file, $rewrite)->getPath();
+ $stl = $this->LDViewService->datToStl($file, $this->rewite)->getPath();
$model->setPath($stl);
$model
@@ -271,12 +272,14 @@ class ModelLoader extends BaseLoader
return $context->getAdapter()->getPathPrefix().$filename;
}
// Try to find model in current LDRAW\PARTS sub-directory
- elseif ($this->ldrawLibraryContext->has('parts/'.$filename)) {
- return $this->ldrawLibraryContext->getAdapter()->getPathPrefix().'parts'.DIRECTORY_SEPARATOR.$filename;
- }
- // Try to find model in current LDRAW\P sub-directory
- elseif ($this->ldrawLibraryContext->has('p'.DIRECTORY_SEPARATOR.$filename)) {
- return $this->ldrawLibraryContext->getAdapter()->getPathPrefix().'p'.DIRECTORY_SEPARATOR.$filename;
+ elseif($this->ldrawLibraryContext) {
+ if($this->ldrawLibraryContext->has('parts/'.$filename)) {
+ return $this->ldrawLibraryContext->getAdapter()->getPathPrefix().'parts'.DIRECTORY_SEPARATOR.$filename;
+ }
+ // Try to find model in current LDRAW\P sub-directory
+ elseif ($this->ldrawLibraryContext->has('p'.DIRECTORY_SEPARATOR.$filename)) {
+ return $this->ldrawLibraryContext->getAdapter()->getPathPrefix().'p'.DIRECTORY_SEPARATOR.$filename;
+ }
}
return null;
@@ -297,6 +300,7 @@ class ModelLoader extends BaseLoader
return new Filesystem($adapter);
} catch (Exception $exception) {
$this->logger->error($exception->getMessage());
+ return null;
}
}
@@ -322,7 +326,6 @@ class ModelLoader extends BaseLoader
// Do not include models without permission to redistribute
elseif ($modelArray['license'] != 'Redistributable under CCAL version 2.0') {
$this->logger->info('Model skipped.', ['number' => $modelArray['id'], 'license' => $modelArray['license']]);
-
return false;
}
diff --git a/src/AppBundle/Service/Loader/RebrickableLoader.php b/src/AppBundle/Service/Loader/RebrickableLoader.php
index 7df7f91..ea68a1b 100644
--- a/src/AppBundle/Service/Loader/RebrickableLoader.php
+++ b/src/AppBundle/Service/Loader/RebrickableLoader.php
@@ -21,7 +21,7 @@ class RebrickableLoader extends BaseLoader
$this->rebrickable_url = $rebrickable_url;
}
- public function loadTables()
+ public function loadAll()
{
$connection = $this->em->getConnection();
$connection->beginTransaction();
@@ -33,9 +33,8 @@ class RebrickableLoader extends BaseLoader
$this->truncateTables();
$connection->prepare('SET foreign_key_checks = 1;')->execute();
- $this->output->writeln([
+ $this->writeOutput([
'Truncated rebrickable database tables.',
- '------------------------------------------------------------------------------',
'Loading CSV files into database...',
]);
@@ -54,7 +53,7 @@ class RebrickableLoader extends BaseLoader
$connection->commit();
- $this->output->writeln('Rebrickable database loaded successfully!');
+ $this->writeOutput(['Rebrickable database loaded successfully!']);
} catch (\Exception $e) {
$connection->rollBack();
throw $e;
@@ -65,19 +64,17 @@ class RebrickableLoader extends BaseLoader
{
$array = ['inventories', 'inventory_parts', 'inventory_sets', 'sets', 'themes', 'parts', 'part_categories', 'colors'];
- $this->output->writeln([
- 'Loading Rebrickable CSV files',
- '------------------------------------------------------------------------------',
+ $this->writeOutput([
+ '------------------------------------------------------------------------------>',
+ 'Loading Rebrickable CSV files>',
+ '------------------------------------------------------------------------------>',
]);
foreach ($array as $item) {
$this->csvFile[$item] = $this->downloadFile($this->rebrickable_url.$item.'.csv');
}
- $this->output->writeln([
- 'Done',
- '------------------------------------------------------------------------------',
- ]);
+ $this->writeOutput(['All CSV files loaded.']);
}
private function truncateTables()
diff --git a/src/AppBundle/Service/Loader/RelationLoader.php b/src/AppBundle/Service/Loader/RelationLoader.php
index 7ceaf73..925fec7 100644
--- a/src/AppBundle/Service/Loader/RelationLoader.php
+++ b/src/AppBundle/Service/Loader/RelationLoader.php
@@ -30,26 +30,21 @@ class RelationLoader extends BaseLoader
public function loadAll()
{
$parts = $this->em->getRepository(Part::class)->findAll();
-
- $this->initProgressBar(count($parts));
- /** @var Part $part */
- foreach ($parts as $part) {
- $this->load($part);
-
- $this->progressBar->advance();
- }
- $this->progressBar->finish();
+ $this->load($parts);
}
- public function loadNotPaired()
+ public function loadNotPaired($parts)
{
$parts = $this->em->getRepository(Part::class)->findAllNotPaired();
+ $this->load($parts);
+ }
+ private function load($parts) {
$this->initProgressBar(count($parts));
/** @var Part $part */
foreach ($parts as $part) {
- $this->load($part);
- $this->progressBar->setMessage($part->getNumber(), 'filename');
+ $this->loadPartRelation($part);
+ $this->progressBar->setMessage($part->getNumber());
$this->progressBar->advance();
}
$this->progressBar->finish();
@@ -62,7 +57,7 @@ class RelationLoader extends BaseLoader
*
* @return Model $m
*/
- private function load(Part $part)
+ private function loadPartRelation(Part $part)
{
$modelRepository = $this->em->getRepository(Model::class);