diff --git a/app/config/config.yml b/app/config/config.yml index 5ae4fd1..4b4bbd4 100644 --- a/app/config/config.yml +++ b/app/config/config.yml @@ -7,6 +7,11 @@ imports: # http://symfony.com/doc/current/best_practices/configuration.html#application-related-configuration parameters: locale: en + rebrickable_url: + pieces: 'http://rebrickable.com/files/pieces.csv.gz' + sets: 'http://rebrickable.com/files/sets.csv.gz' + set_pieces: 'http://rebrickable.com/files/set_pieces.csv.gz' + ldraw_url: 'http://www.ldraw.org/library/updates/completeCA.zip' framework: #esi: ~ diff --git a/app/config/services.yml b/app/config/services.yml index 028f1c7..5814f07 100644 --- a/app/config/services.yml +++ b/app/config/services.yml @@ -18,11 +18,11 @@ services: arguments: ['@doctrine.orm.entity_manager', '@manager.brickset','@manager.rebrickable'] loader.rebrickable: - class: AppBundle\Loader\RebrickableLoader - arguments: ['@doctrine.orm.entity_manager', '@manager.rebrickable'] + class: AppBundle\Command\Loader\RebrickableLoader + arguments: ['@doctrine.orm.entity_manager', '@manager.rebrickable', '%rebrickable_url%' ] loader.ldraw: - class: AppBundle\Loader\LDrawLoader - arguments: ['@doctrine.orm.entity_manager', '%kernel.root_dir%/../bin/ldview', '@oneup_flysystem.ldraw_filesystem'] + class: AppBundle\Command\Loader\LDrawLoader + arguments: ['@doctrine.orm.entity_manager', '%kernel.root_dir%/../bin/ldview', '@oneup_flysystem.ldraw_filesystem', '%ldraw_url%'] app.form.filter_set: class: AppBundle\Form\FilterSetType diff --git a/bin/ldview b/bin/ldview index 6f2efe9..d255b0f 100644 Binary files a/bin/ldview and b/bin/ldview differ diff --git a/src/AppBundle/Command/LoadLDrawCommand.php b/src/AppBundle/Command/LoadLDrawCommand.php deleted file mode 100644 index 464054f..0000000 --- a/src/AppBundle/Command/LoadLDrawCommand.php +++ /dev/null @@ -1,29 +0,0 @@ -setName('app:load:ldraw') - ->setDescription('Loads LDraw library parts') - ->setHelp('This command allows you to..') - ->addArgument('ldraw', InputArgument::REQUIRED, 'Path to LDraw library folder'); - } - - protected function execute(InputInterface $input, OutputInterface $output) - { - $ldrawLoader = $this->getContainer()->get('loader.ldraw'); - - $ldrawLoader->setOutput($output); - - $ldrawLoader->loadModels($input->getArgument('ldraw')); - } -} diff --git a/src/AppBundle/Command/LoadRebrickableCommand.php b/src/AppBundle/Command/LoadLibraryCommand.php similarity index 59% rename from src/AppBundle/Command/LoadRebrickableCommand.php rename to src/AppBundle/Command/LoadLibraryCommand.php index 859d0d5..d96d211 100644 --- a/src/AppBundle/Command/LoadRebrickableCommand.php +++ b/src/AppBundle/Command/LoadLibraryCommand.php @@ -3,26 +3,32 @@ namespace AppBundle\Command; use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand; +use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; -class LoadRebrickableCommand extends ContainerAwareCommand +class LoadLibraryCommand extends ContainerAwareCommand { protected function configure() { $this - ->setName('app:load:rebrickable') - ->setDescription('Loads Rebrickable csv data') - ->setHelp('This command allows you to..'); + ->setName('app:load:library') + ->setDescription('Loads LDraw library parts') + ->setHelp('This command allows you to..') + ->addArgument('ldraw', InputArgument::OPTIONAL, 'Path to LDraw library folder'); } protected function execute(InputInterface $input, OutputInterface $output) { - $rebrickableLoader = $this->getContainer()->get('loader.rebrickable'); + $ldrawLoader = $this->getContainer()->get('loader.ldraw'); + $ldrawLoader->setOutput($output); + $rebrickableLoader = $this->getContainer()->get('loader.rebrickable'); $rebrickableLoader->setOutput($output); try { + $ldrawLoader->loadModels($input->getArgument('ldraw')); + $rebrickableLoader->loadColors(); $rebrickableLoader->loadParts(); diff --git a/src/AppBundle/Loader/LDrawLoader.php b/src/AppBundle/Command/Loader/LDrawLoader.php similarity index 68% rename from src/AppBundle/Loader/LDrawLoader.php rename to src/AppBundle/Command/Loader/LDrawLoader.php index 228648a..92dc37f 100644 --- a/src/AppBundle/Loader/LDrawLoader.php +++ b/src/AppBundle/Command/Loader/LDrawLoader.php @@ -1,6 +1,6 @@ em = $em; $this->ldview = $ldview; $this->dataPath = $dataPath; + $this->ldraw_url = $ldraw_url; + } + + public function downloadLibrary() + { + $this->output->writeln('Downloading set_pieces.csv from Rebrickable.com'); + $temp = $this->downloadFile($this->ldraw_url); + $temp_dir = tempnam(sys_get_temp_dir(), 'printabrick.'); + if (file_exists($temp_dir)) { + unlink($temp_dir); + } + mkdir($temp_dir); + $zip = new \ZipArchive(); + if ($zip->open($temp) != 'true') { + echo 'Error :- Unable to open the Zip File'; + } + $zip->extractTo($temp_dir); + $zip->close(); + unlink($temp); + + return $temp_dir; } public function loadModels($LDrawLibrary) @@ -85,9 +107,16 @@ class LDrawLoader extends Loader if ($line !== '') { $line = preg_replace('/^0 /', '', $line); - // 0 - if (!$firstLine) { //TODO handle ~Moved to - $category = explode(' ', trim($line))[0]; + // 0 + if (!$firstLine) { + //TODO handle "~Moved to" + //TODO "=" - alias name for other part kept for referece + //TODO "_" shortcut + + $array = explode(' ', trim($line), 2); + $category = isset($array[0]) ? $array[0] : ''; + $model->setName($line); + $firstLine = true; } // 0 !CATEGORY @@ -113,8 +142,6 @@ class LDrawLoader extends Loader if ($cat == null) { $cat = new Category(); $cat->setName($category); - $this->em->persist($cat); - $this->em->flush(); } $model->setCategory($cat); @@ -134,25 +161,25 @@ class LDrawLoader extends Loader */ private function createStlFile($file) { - $builder = new ProcessBuilder(); - $process = $builder - ->setPrefix($this->ldview) - ->setArguments([ -// $this->ldraw->getAdapter()->getPathPrefix().$file['path'], - $file->getRealPath(), - '-ExportFiles=1', - '-LDrawDir='.$this->ldraw->getAdapter()->getPathPrefix(), - '-ExportSuffix=.stl', - '-ExportsDir='.$this->dataPath->getAdapter()->getPathPrefix(), - ]) - ->getProcess(); - - $process->run(); - $stlFilename = str_replace('.dat', '.stl', $file->getFilename()); - if (!$process->isSuccessful() || !$this->dataPath->has($stlFilename)) { - throw new ProcessFailedException($process); //TODO + if (!$this->dataPath->has($stlFilename)) { + $builder = new ProcessBuilder(); + $process = $builder + ->setPrefix($this->ldview) + ->setArguments([ +// $this->ldraw->getAdapter()->getPathPrefix().$file['path'], + $file->getRealPath(), + '-LDrawDir='.$this->ldraw->getAdapter()->getPathPrefix(), + '-ExportFile='.$this->dataPath->getAdapter()->getPathPrefix().$stlFilename, + ]) + ->getProcess(); + + $process->run(); + + if (!$process->isSuccessful() || !$this->dataPath->has($stlFilename)) { + throw new LogicException($file->getFilename().' : '.$process->getOutput()); //TODO + } } return $this->dataPath->get($stlFilename); diff --git a/src/AppBundle/Command/Loader/Loader.php b/src/AppBundle/Command/Loader/Loader.php new file mode 100644 index 0000000..f0f2622 --- /dev/null +++ b/src/AppBundle/Command/Loader/Loader.php @@ -0,0 +1,72 @@ +output = $output; + $this->output->setDecorated(true); + } + + private function progressCallback($notification_code, $severity, $message, $message_code, $bytes_transferred, $bytes_max) + { + switch ($notification_code) { + case STREAM_NOTIFY_FILE_SIZE_IS: + $this->progressBar = new ProgressBar($this->output); + $this->progressBar->setBarWidth(100); + $this->progressBar->start($bytes_max); + break; + case STREAM_NOTIFY_PROGRESS: + $this->progressBar->setProgress($bytes_transferred); + break; + case STREAM_NOTIFY_COMPLETED: + $this->progressBar->setProgress($bytes_transferred); + $this->progressBar->finish(); + break; + } + } + + protected function downloadFile($url) + { + $temp = tempnam(sys_get_temp_dir(), 'printabrick.'); + + $ctx = stream_context_create([], [ + 'notification' => [$this, 'progressCallback'], + ]); + + try { + if (false === file_put_contents($temp, fopen($url, 'r', 0, $ctx))) { + throw new LogicException('error writing file'); //TODO + } + } catch (ContextErrorException $e) { + throw new LogicException('wrong url'); //TODO + } catch (\Exception $e) { + throw new LogicException('exception: '.$e->getMessage()); //TODO + } + + return $temp; + } +} diff --git a/src/AppBundle/Loader/RebrickableLoader.php b/src/AppBundle/Command/Loader/RebrickableLoader.php similarity index 80% rename from src/AppBundle/Loader/RebrickableLoader.php rename to src/AppBundle/Command/Loader/RebrickableLoader.php index e648a8f..2009394 100644 --- a/src/AppBundle/Loader/RebrickableLoader.php +++ b/src/AppBundle/Command/Loader/RebrickableLoader.php @@ -1,6 +1,6 @@ em = $em; $this->rebrickableManager = $rebrickableManager; + $this->rebrickable_url = $rebrickable_url; } public function loadPartBuildingKits() { + $this->output->writeln('Downloading set_pieces.csv from Rebrickable.com'); + $file = $this->downloadFile('compress.zlib://'.$this->rebrickable_url['set_pieces']); + $partRepository = $this->em->getRepository('AppBundle:Part'); $buldingKitRepository = $this->em->getRepository('AppBundle:BuildingKit'); $colorRepository = $this->em->getRepository('AppBundle:Color'); - $setPieces = tempnam(sys_get_temp_dir(), 'printabrick.'); - file_put_contents($setPieces, fopen('compress.zlib://http://rebrickable.com/files/set_pieces.csv.gz', 'r')); - $this->em->getConnection()->getConfiguration()->setSQLLogger(null); - if (($handle = fopen($setPieces, 'r')) !== false) { + $this->output->writeln('Loading set_pieces.csv into Database'); + if (($handle = fopen($file, 'r')) !== false) { $header = fgetcsv($handle, 200, ','); // create a new progress bar (50 units) - $progress = new ProgressBar($this->output, intval(exec("wc -l '$setPieces'"))); //TODO replace wc-l + $progress = new ProgressBar($this->output, intval(exec("wc -l '$file'"))); //TODO replace wc-l $progress->setFormat('very_verbose'); $progress->setBarWidth(50); $progress->start(); @@ -83,26 +83,26 @@ class RebrickableLoader extends Loader $this->em->clear(); fclose($handle); $progress->finish(); - $progress->clear(); } - unlink($setPieces); + unlink($file); } public function loadBuildingKits() { - $keywordRepository = $this->em->getRepository('AppBundle:Keyword'); + $this->output->writeln('Downloading sets.csv from Rebrickable.com'); + $file = $this->downloadFile('compress.zlib://'.$this->rebrickable_url['sets']); - $sets = tempnam(sys_get_temp_dir(), 'printabrick.'); - file_put_contents($sets, fopen('compress.zlib://http://rebrickable.com/files/sets.csv.gz', 'r')); + $keywordRepository = $this->em->getRepository('AppBundle:Keyword'); $this->em->getConnection()->getConfiguration()->setSQLLogger(null); - if (($handle = fopen($sets, 'r')) !== false) { + $this->output->writeln('Loading sets.csv into Database'); + if (($handle = fopen($file, 'r')) !== false) { $header = fgetcsv($handle, 500, ','); // create a new progress bar (50 units) - $progress = new ProgressBar($this->output, intval(exec("wc -l '$sets'"))); //TODO replace wc-l + $progress = new ProgressBar($this->output, intval(exec("wc -l '$file'"))); //TODO replace wc-l $progress->setFormat('very_verbose'); $progress->setBarWidth(50); $progress->start(); @@ -145,23 +145,23 @@ class RebrickableLoader extends Loader fclose($handle); $progress->finish(); - $progress->clear(); } - unlink($sets); + unlink($file); } public function loadParts() { - $pieces = tempnam(sys_get_temp_dir(), 'printabrick.'); - file_put_contents($pieces, fopen('compress.zlib://http://rebrickable.com/files/pieces.csv.gz', 'r')); //TODO replace wc-l + $this->output->writeln('Downloading pieces.csv from Rebrickable.com'); + $file = $this->downloadFile('compress.zlib://'.$this->rebrickable_url['pieces']); $categoryRepository = $this->em->getRepository('AppBundle:Category'); $this->em->getConnection()->getConfiguration()->setSQLLogger(null); - if (($handle = fopen($pieces, 'r')) !== false) { + $this->output->writeln('Loading pieces.csv into Database'); + if (($handle = fopen($file, 'r')) !== false) { // create a new progress bar (50 units) - $progress = new ProgressBar($this->output, intval(exec("wc -l '$pieces'"))); + $progress = new ProgressBar($this->output, intval(exec("wc -l '$file'"))); //TODO replace wc-l $progress->setFormat('very_verbose'); $progress->setBarWidth(50); $progress->start(); @@ -194,14 +194,15 @@ class RebrickableLoader extends Loader fclose($handle); $progress->finish(); - $progress->clear(); } - unlink($pieces); + unlink($file); } public function loadColors() { + $this->output->writeln('Loading colors into Database'); + $rb_colors = $this->rebrickableManager->getColors(); foreach ($rb_colors as $rb_color) { @@ -221,10 +222,10 @@ class RebrickableLoader extends Loader { $modelRepository = $this->em->getRepository('AppBundle:Model'); - if (strpos($part->getNumber(), 'p')) { + $model = $modelRepository->findOneBy(['number' => $part->getNumber()]); + + if (!$model && strpos($part->getNumber(), 'p')) { $model = $modelRepository->findOneBy(['number' => explode('p', $part->getNumber())[0]]); - } else { - $model = $modelRepository->findOneBy(['number' => $part->getNumber()]); } return $model; diff --git a/src/AppBundle/Loader/Loader.php b/src/AppBundle/Loader/Loader.php deleted file mode 100644 index d5e0325..0000000 --- a/src/AppBundle/Loader/Loader.php +++ /dev/null @@ -1,22 +0,0 @@ -output = $output; - $this->output->setDecorated(true); - } -}