v/PrintABrick
1
0
Fork 0
mirror of https://github.com/ToxicCrack/PrintABrick.git synced 2025-05-18 05:10:07 -07:00
Code Issues Projects Releases Wiki Activity

Add initial load command

Browse Source
This commit is contained in:
David Hübner 2017-04-21 02:14:07 +02:00
parent b8f4b77bd2
commit efde576d1c
10 changed files with 163 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
app/config/service/loader.yml
View File

@ -3,11 +3,7 @@ services:
abstract: true
class: AppBundle\Service\Loader\BaseLoader
calls:
- [setArguments, ['@doctrine.orm.entity_manager', '@monolog.logger.loader']]
service.ldview:
class: AppBundle\Service\LDViewService
arguments: ['%ldview_bin%', '@oneup_flysystem.media_filesystem']
- [setArguments, ['@doctrine.orm.entity_manager', '@monolog.logger.loader', '@app.transformer.format']]
service.loader.rebrickable:
class: AppBundle\Service\Loader\RebrickableLoader

4
app/config/service/service.yml
View File

@ -14,3 +14,7 @@ services:
service.renderer.stl:
class: AppBundle\Service\StlRendererService
arguments: ['%kernel.root_dir%/Resources/povray_layout/layout.tmpl', '%povray_bin%','%stl2pov_bin%']
service.ldview:
class: AppBundle\Service\LDViewService
arguments: ['%ldview_bin%', '@oneup_flysystem.media_filesystem']

67
src/AppBundle/Command/InitDataCommand.php Normal file
View File

@ -0,0 +1,67 @@
<?php
namespace AppBundle\Command;
use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
use Symfony\Component\Console\Input\ArrayInput;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputDefinition;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
class InitDataCommand extends ContainerAwareCommand
{
protected function configure()
{
$this
->setName('app:init')
->setDescription('Loads relations between LDraw models and Rebrickable parts.')
->setHelp('This command allows you to load relation between models and parts into database.')
->setDefinition(
new InputDefinition([
new InputArgument('ldraw', InputArgument::REQUIRED, 'Path to LDraw library directory'),
])
);
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$loadModelsCommand = $this->getApplication()->find('app:load:models');
$returnCode = $loadModelsCommand->run(new ArrayInput([
'command' => 'app:load:models',
'ldraw' => $input->getArgument('ldraw'),
'--all' => true
]),$output);
if($returnCode) {
return 1;
}
$loadRebrickableCommad = $this->getApplication()->find('app:load:rebrickable');
$returnCode = $loadRebrickableCommad->run(new ArrayInput([
'command' => 'app:load:rebrickable']),$output);
if($returnCode) {
return 1;
}
$loadRelationsCommand = $this->getApplication()->find('app:load:relations');
$returnCode = $loadRelationsCommand->run(new ArrayInput([
'command' => 'app:load:relations']),$output);
if($returnCode) {
return 1;
}
$loadImagesCommand = $this->getApplication()->find('app:load:images');
$returnCode = $loadImagesCommand->run(new ArrayInput([
'command' => 'app:load:images',
'--color' => -1,
'--rebrickable' => true,
'--models' => true
]),$output);
}
}

20
src/AppBundle/Command/LoadModelsCommand.php
View File

@ -23,7 +23,6 @@ class LoadModelsCommand extends ContainerAwareCommand
->setDefinition(
new InputDefinition([
new InputArgument('ldraw', InputArgument::REQUIRED, 'Path to LDraw library directory'),
// new InputOption('images', 'i', InputOption::VALUE_NONE, 'Do you want to generate images of models?'),
new InputOption('all', 'a', InputOption::VALUE_NONE, 'Load all models from LDraw libary folder (/parts directory)'),
new InputOption('file', 'f', InputOption::VALUE_REQUIRED, 'Load single modle into database'),
new InputOption('update', 'u', InputOption::VALUE_NONE, 'Overwrite already loaded models'),
@ -35,8 +34,7 @@ class LoadModelsCommand extends ContainerAwareCommand
{
if (!$this->lock()) {
$output->writeln('The command is already running in another process.');
return 0;
return 1;
}
$modelLoader = $this->getContainer()->get('service.loader.model');
@ -51,12 +49,10 @@ class LoadModelsCommand extends ContainerAwareCommand
if (($path = $input->getOption('file')) != null) {
if ($file = realpath($path)) {
$output->writeln([
'Loading model',
'path: '.$file,
'------------------------------------------------------------------------------',
"Loading model: {$path}",
]);
$modelLoader->loadOneModel($file);
$modelLoader->loadOne($file);
$errorCount = $this->getContainer()->get('monolog.logger.loader')->countErrors();
$errors = $errorCount ? '<error>'.$errorCount.'</error>' : '<info>0</info>';
@ -70,10 +66,12 @@ class LoadModelsCommand extends ContainerAwareCommand
// Load all models inside ldraw/parts directory
if ($input->getOption('all')) {
$output->writeln([
'Loading models from LDraw library: <comment>'.$ldrawPath.'</comment>',
'<fg=cyan>------------------------------------------------------------------------------</>',
"<fg=cyan>Loading models from LDraw library:</> <comment>{$ldrawPath}</comment>",
'<fg=cyan>------------------------------------------------------------------------------</>',
]);
$modelLoader->loadAllModels();
$modelLoader->loadAll();
$errorCount = $this->getContainer()->get('monolog.logger.loader')->countErrors();
$errors = $errorCount ? '<error>'.$errorCount.'</error>' : '<info>0</info>';
@ -81,7 +79,9 @@ class LoadModelsCommand extends ContainerAwareCommand
$output->writeln(['Done with "'.$errors.'" errors.']);
}
} else {
$output->writeln($ldraw.' is not valid path');
$output->writeln("<error>{$ldraw} is not a valid path!</error>");
$this->release();
return 1;
}
$this->release();

9
src/AppBundle/Command/LoadRebrickableDataCommand.php
View File

@ -2,6 +2,7 @@
namespace AppBundle\Command;
use League\Flysystem\Exception;
use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
@ -21,7 +22,11 @@ class LoadRebrickableDataCommand extends ContainerAwareCommand
$rebrickableLoader = $this->getContainer()->get('service.loader.rebrickable');
$rebrickableLoader->setOutput($output);
//TODO log errors
$rebrickableLoader->loadTables();
try {
$rebrickableLoader->loadAll();
} catch (Exception $exception) {
$output->writeln("<error>{$exception->getMessage()}</error>");
return 1;
}
}
}

12
src/AppBundle/Command/LoadRelationCommand.php
View File

@ -21,7 +21,15 @@ class LoadRelationCommand extends ContainerAwareCommand
$relationLoader = $this->getContainer()->get('service.loader.relation');
$relationLoader->setOutput($output);
//TODO log errors
$relationLoader->loadNotPaired();
$output->writeln([
'<fg=cyan>------------------------------------------------------------------------------</>',
"<fg=cyan>Loading relations between parts and models...</>",
'<fg=cyan>------------------------------------------------------------------------------</>',
]);
$relationLoader->loadAll();
$output->writeln(['<info>Done!</info>']);
}
}

36
src/AppBundle/Service/Loader/BaseLoader.php
View File

@ -4,6 +4,7 @@ namespace AppBundle\Service\Loader;
use AppBundle\Exception\FileNotFoundException;
use AppBundle\Exception\WriteErrorException;
use AppBundle\Transformer\FormatTransformer;
use Doctrine\ORM\EntityManager;
use Monolog\Logger;
use Symfony\Bundle\FrameworkBundle\Translation\Translator;
@ -29,21 +30,29 @@ abstract class BaseLoader
*/
protected $progressBar;
/** @var Logger */
/**
* @var Logger
*/
protected $logger;
/**
* @var FormatTransformer
*/
private $formatTransformer;
/**
* Loader constructor.
*
* @param EntityManager $em
* @param Translator $translator
*/
public function setArguments(EntityManager $em, $logger)
public function setArguments(EntityManager $em, $logger, $formatTransformer)
{
$this->em = $em;
$this->em->getConnection()->getConfiguration()->setSQLLogger(null);
$this->logger = $logger;
$this->formatTransformer = $formatTransformer;
}
public function setOutput(OutputInterface $output)
@ -57,28 +66,33 @@ abstract class BaseLoader
*
* @param $total
*/
protected function initProgressBar($total)
protected function initProgressBar($total, $format = 'loader')
{
$this->progressBar = new ProgressBar($this->output, $total);
// $this->progressBar->setFormat('very_verbose');
$this->progressBar->setFormat('[%current%/%max%] [%bar%] %percent:3s%% (%elapsed:6s%/%estimated:-6s%) (%filename%)'.PHP_EOL);
ProgressBar::setFormatDefinition('loader', '[%current% / %max%] [%bar%] %percent:3s%% (%elapsed:6s%/%estimated:-6s%) (%message%)' . PHP_EOL);
ProgressBar::setFormatDefinition('download', '[%progress% / %size%] [%bar%] %percent:3s%% (%elapsed:6s%/%estimated:-6s%)' . PHP_EOL);
$this->progressBar->setFormat($format);
$this->progressBar->setBarWidth(70);
$this->progressBar->start();
}
protected function progressCallback($notification_code, $severity, $message, $message_code, $bytes_transferred, $bytes_max)
{
$this->logger->info($notification_code);
switch ($notification_code) {
case STREAM_NOTIFY_FILE_SIZE_IS:
$this->initProgressBar($bytes_max);
$this->progressBar->setFormat('[%current%/%max%] [%bar%] %percent:3s%% (%elapsed:6s%/%estimated:-6s%)'.PHP_EOL);
$this->progressBar->setFormat('download');
$this->progressBar->setMessage($this->formatTransformer->bytesToSize($bytes_max),'size');
$this->progressBar->setRedrawFrequency($bytes_max/20);
break;
case STREAM_NOTIFY_PROGRESS:
$this->progressBar->setProgress($bytes_transferred);
$this->progressBar->setMessage($this->formatTransformer->bytesToSize($bytes_transferred),'progress');
break;
case STREAM_NOTIFY_COMPLETED:
$this->progressBar->setMessage('<info>Done</info>');
$this->progressBar->setProgress($bytes_max);
$this->progressBar->finish();
break;
}
@ -106,6 +120,7 @@ abstract class BaseLoader
if (false === file_put_contents($temp, fopen($url, 'r', 0, $ctx))) {
throw new WriteErrorException($temp);
}
$this->progressBar->finish();
} catch (ContextErrorException $e) {
throw new FileNotFoundException($url);
} catch (\Exception $e) {
@ -114,4 +129,9 @@ abstract class BaseLoader
return $temp;
}
protected function writeOutput(array $lines) {
if($this->output)
$this->output->writeln($lines);
}
}

37
src/AppBundle/Service/Loader/ModelLoader.php
View File

@ -12,15 +12,14 @@ use AppBundle\Exception\ConvertingFailedException;
use AppBundle\Exception\FileException;
use AppBundle\Exception\ParseErrorException;
use AppBundle\Service\LDViewService;
use AppBundle\Utils\LDModelParser;
use AppBundle\Utils\RelationMapper;
use AppBundle\Util\LDModelParser;
use AppBundle\Util\RelationMapper;
use League\Flysystem\Adapter\Local;
use League\Flysystem\Exception;
use League\Flysystem\Filesystem;
use Symfony\Component\Finder\Finder;
use Symfony\Component\Finder\SplFileInfo;
//TODO refactor
class ModelLoader extends BaseLoader
{
/**
@ -80,7 +79,7 @@ class ModelLoader extends BaseLoader
}
}
public function loadOneModel($file)
public function loadOne($file)
{
$connection = $this->em->getConnection();
try {
@ -95,7 +94,7 @@ class ModelLoader extends BaseLoader
}
}
public function loadAllModels()
public function loadAll()
{
$files = $this->finder->in([
$this->ldrawLibraryContext->getAdapter()->getPathPrefix(),
@ -109,7 +108,7 @@ class ModelLoader extends BaseLoader
$connection->beginTransaction();
try {
$this->progressBar->setMessage($file->getFilename(), 'filename');
$this->progressBar->setMessage($file->getFilename());
$this->loadModel($file->getRealPath());
@ -127,7 +126,8 @@ class ModelLoader extends BaseLoader
}
/**
* Load Model entity into database.
* Load model entity and all related submodels into database while generating stl file of model.
* Uses LDView to convert LDraw .dat to .stl
*
* @param $file
*
@ -144,7 +144,8 @@ class ModelLoader extends BaseLoader
// Parse model file save data to $modelArray
try {
$modelArray = $this->ldModelParser->parse($file);
$content = file_get_contents($file);
$modelArray = $this->ldModelParser->parse($content);
} catch (ParseErrorException $e) {
$this->logger->error($e->getMessage(), [$file]);
@ -172,8 +173,8 @@ class ModelLoader extends BaseLoader
return $parentModel;
}
// Load model
// Load model
$model = $this->em->getRepository(Model::class)->getOrCreate($modelArray['id']);
// Recursively load models of subparts
@ -203,7 +204,7 @@ class ModelLoader extends BaseLoader
try {
// update model only if newer version
if (!$model->getModified() || ($model->getModified() < $modelArray['modified'])) {
$stl = $this->LDViewService->datToStl($file, $rewrite)->getPath();
$stl = $this->LDViewService->datToStl($file, $this->rewite)->getPath();
$model->setPath($stl);
$model
@ -271,12 +272,14 @@ class ModelLoader extends BaseLoader
return $context->getAdapter()->getPathPrefix().$filename;
}
// Try to find model in current LDRAW\PARTS sub-directory
elseif ($this->ldrawLibraryContext->has('parts/'.$filename)) {
return $this->ldrawLibraryContext->getAdapter()->getPathPrefix().'parts'.DIRECTORY_SEPARATOR.$filename;
}
// Try to find model in current LDRAW\P sub-directory
elseif ($this->ldrawLibraryContext->has('p'.DIRECTORY_SEPARATOR.$filename)) {
return $this->ldrawLibraryContext->getAdapter()->getPathPrefix().'p'.DIRECTORY_SEPARATOR.$filename;
elseif($this->ldrawLibraryContext) {
if($this->ldrawLibraryContext->has('parts/'.$filename)) {
return $this->ldrawLibraryContext->getAdapter()->getPathPrefix().'parts'.DIRECTORY_SEPARATOR.$filename;
}
// Try to find model in current LDRAW\P sub-directory
elseif ($this->ldrawLibraryContext->has('p'.DIRECTORY_SEPARATOR.$filename)) {
return $this->ldrawLibraryContext->getAdapter()->getPathPrefix().'p'.DIRECTORY_SEPARATOR.$filename;
}
}
return null;
@ -297,6 +300,7 @@ class ModelLoader extends BaseLoader
return new Filesystem($adapter);
} catch (Exception $exception) {
$this->logger->error($exception->getMessage());
return null;
}
}
@ -322,7 +326,6 @@ class ModelLoader extends BaseLoader
// Do not include models without permission to redistribute
elseif ($modelArray['license'] != 'Redistributable under CCAL version 2.0') {
$this->logger->info('Model skipped.', ['number' => $modelArray['id'], 'license' => $modelArray['license']]);
return false;
}

19
src/AppBundle/Service/Loader/RebrickableLoader.php
View File

@ -21,7 +21,7 @@ class RebrickableLoader extends BaseLoader
$this->rebrickable_url = $rebrickable_url;
}
public function loadTables()
public function loadAll()
{
$connection = $this->em->getConnection();
$connection->beginTransaction();
@ -33,9 +33,8 @@ class RebrickableLoader extends BaseLoader
$this->truncateTables();
$connection->prepare('SET foreign_key_checks = 1;')->execute();
$this->output->writeln([
$this->writeOutput([
'<info>Truncated</info> <comment>rebrickable</comment> <info>database tables.</info>',
'------------------------------------------------------------------------------',
'Loading CSV files into database...',
]);
@ -54,7 +53,7 @@ class RebrickableLoader extends BaseLoader
$connection->commit();
$this->output->writeln('Rebrickable database loaded successfully!');
$this->writeOutput(['Rebrickable database loaded successfully!']);
} catch (\Exception $e) {
$connection->rollBack();
throw $e;
@ -65,19 +64,17 @@ class RebrickableLoader extends BaseLoader
{
$array = ['inventories', 'inventory_parts', 'inventory_sets', 'sets', 'themes', 'parts', 'part_categories', 'colors'];
$this->output->writeln([
'Loading Rebrickable CSV files',
'------------------------------------------------------------------------------',
$this->writeOutput([
'<fg=cyan>------------------------------------------------------------------------------</>',
'<fg=cyan>Loading Rebrickable CSV files</>',
'<fg=cyan>------------------------------------------------------------------------------</>',
]);
foreach ($array as $item) {
$this->csvFile[$item] = $this->downloadFile($this->rebrickable_url.$item.'.csv');
}
$this->output->writeln([
'<info>Done</info>',
'------------------------------------------------------------------------------',
]);
$this->writeOutput(['<info>All CSV files loaded.</info>']);
}
private function truncateTables()

21
src/AppBundle/Service/Loader/RelationLoader.php
View File

@ -30,26 +30,21 @@ class RelationLoader extends BaseLoader
public function loadAll()
{
$parts = $this->em->getRepository(Part::class)->findAll();
$this->initProgressBar(count($parts));
/** @var Part $part */
foreach ($parts as $part) {
$this->load($part);
$this->progressBar->advance();
}
$this->progressBar->finish();
$this->load($parts);
}
public function loadNotPaired()
public function loadNotPaired($parts)
{
$parts = $this->em->getRepository(Part::class)->findAllNotPaired();
$this->load($parts);
}
private function load($parts) {
$this->initProgressBar(count($parts));
/** @var Part $part */
foreach ($parts as $part) {
$this->load($part);
$this->progressBar->setMessage($part->getNumber(), 'filename');
$this->loadPartRelation($part);
$this->progressBar->setMessage($part->getNumber());
$this->progressBar->advance();
}
$this->progressBar->finish();
@ -62,7 +57,7 @@ class RelationLoader extends BaseLoader
*
* @return Model $m
*/
private function load(Part $part)
private function loadPartRelation(Part $part)
{
$modelRepository = $this->em->getRepository(Model::class);