Skip to content

Commit

Permalink
Merge branch 'release/0.5'
Browse files Browse the repository at this point in the history
  • Loading branch information
IllyaMoskvin committed Jan 11, 2018
2 parents 67fe37f + d61f722 commit 0567f65
Show file tree
Hide file tree
Showing 127 changed files with 5,284 additions and 3,090 deletions.
8 changes: 7 additions & 1 deletion .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -35,14 +35,20 @@ SCOUT_QUEUE=true
ELASTICSEARCH_HOST=localhost
ELASTICSEARCH_PORT=9200
ELASTICSEARCH_SCHEME=http
ELASTICSEARCH_INDEX=data_aggregator
ELASTICSEARCH_INDEX=test-current # will be prepended to per-model index names, e.g. test-current-artworks
ELASTICSEARCH_ALIAS=test-v1 # use this to search across all model indexes

LAKE_URL=
IIIF_URL=

COLLECTIONS_DATA_SERVICE_URL=
EVENTS_DATA_SERVICE_URL=
DSC_DATA_SERVICE_URL=
ULAN_DATA_SERVICE_URL=
LIBRARY_DATA_SERVICE_URL=
ARCHIVES_DATA_SERVICE_URL=

LEGACY_EVENTS_JSON=
STATIC_ARCHIVE_JSON=

PRIMO_API_SOURCE=
15 changes: 15 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,20 @@
Data Aggregator Changelog
=============================
0.5 - Library, Archives, dominant colors and improve search

* Add Etags to all API output
* Add dominant color information to all images
* Separate Elasticsearch index into separate indexes per resource, for future compatibility
* Refactor agents to store all in a single table, while still providing separate endpoints by type
* Tie Agents to their corresponding ULAN URIs
* Add ability to sort search results
* Abstract portions of code common between APIs into the `data-hub-foundation` package
* Add Library Terms and Materials data to the aggregator
* Add images from the Ryerson & Burnham Library Image Archive to the aggregator
* Add missing relationships to Artwork endpoint—DSC Sections, Mobile Tour Stops, dominant color of preferred image
* Add missing relationships to Artists endpoint—Artworks
* Add missing relationships to Exhibitions endpoint—Events

0.4 - Digital Catalogues, static sites and mobile

* Refactor artists endpoint to return all agents that are marked as a creator for an artwork
Expand Down
2 changes: 1 addition & 1 deletion VERSION
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0.4
0.5
47 changes: 0 additions & 47 deletions app/Console/Commands/AliasSearch.php

This file was deleted.

10 changes: 7 additions & 3 deletions app/Console/Commands/CreateEndpointDocs.php
Original file line number Diff line number Diff line change
Expand Up @@ -54,12 +54,9 @@ public function handle()
}

$doc = '';

$doc .= "# Collections\n\n";
$doc .= \App\Models\Collections\Artwork::instance()->docEndpoints($this->appUrl);
$doc .= \App\Models\Collections\Agent::instance()->docEndpoints($this->appUrl);
$doc .= \App\Models\Collections\Artist::instance()->docEndpoints($this->appUrl);
$doc .= \App\Models\Collections\CorporateBody::instance()->docEndpoints($this->appUrl);
$doc .= \App\Models\Collections\Department::instance()->docEndpoints($this->appUrl);
$doc .= \App\Models\Collections\ObjectType::instance()->docEndpoints($this->appUrl);
$doc .= \App\Models\Collections\Category::instance()->docEndpoints($this->appUrl);
Expand Down Expand Up @@ -92,6 +89,13 @@ public function handle()
$doc .= "# Static Archive\n\n";
$doc .= \App\Models\StaticArchive\Site::instance()->docEndpoints($this->appUrl);

$doc .= "# Archive\n\n";
$doc .= \App\Models\Archive\ArchiveImage::instance()->docEndpoints($this->appUrl);

$doc .= "# Library\n\n";
$doc .= \App\Models\Library\Material::instance()->docEndpoints($this->appUrl);
$doc .= \App\Models\Library\Term::instance()->docEndpoints($this->appUrl);

$doc .= "> Generated by `php artisan docs:endpoints` on " .Carbon::now() ."\n";

Storage::disk('local')->put('ENDPOINTS.md', $doc);
Expand Down
9 changes: 7 additions & 2 deletions app/Console/Commands/CreateFieldsDocs.php
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,6 @@ public function handle()
$doc .= "# Collections\n\n";
$doc .= \App\Models\Collections\Artwork::instance()->docFields();
$doc .= \App\Models\Collections\Agent::instance()->docFields();
$doc .= \App\Models\Collections\Artist::instance()->docFields();
$doc .= \App\Models\Collections\CorporateBody::instance()->docFields();
$doc .= \App\Models\Collections\Department::instance()->docFields();
$doc .= \App\Models\Collections\ObjectType::instance()->docFields();
$doc .= \App\Models\Collections\Category::instance()->docFields();
Expand Down Expand Up @@ -69,6 +67,13 @@ public function handle()
$doc .= "# Static Archive\n\n";
$doc .= \App\Models\StaticArchive\Site::instance()->docFields();

$doc .= "# Archive\n\n";
$doc .= \App\Models\Archive\ArchiveImage::instance()->docFields();

$doc .= "# Library\n\n";
$doc .= \App\Models\Library\Material::instance()->docFields();
$doc .= \App\Models\Library\Term::instance()->docFields();

$doc .= "> Generated by `php artisan docs:fields` on " .Carbon::now() ."\n";

Storage::disk('local')->put('FIELDS.md', $doc);
Expand Down
184 changes: 184 additions & 0 deletions app/Console/Commands/ImagesColor.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,184 @@
<?php

namespace App\Console\Commands;

use Illuminate\Console\Command;
use Illuminate\Support\Facades\Storage;

use Intervention\Image\ImageManager;
use marijnvdwerf\palette\Palette;

use App\Models\Collections\Image;

class ImagesColor extends Command
{

protected $signature = "images:color
{start? : Manual offset for multi processing. Use with force }
{count? : Manual count for multi processing. Use with force }
{--force : Don't skip images that have a dominant color on record already }";

protected $description = 'Determine dominant color for each image';

public function handle()
{

ini_set("memory_limit", "-1");

$files = Storage::files( 'images' );

$files = collect( $files );

// Ignore any file that's not a jpg
$files = $files->filter( function( $file ) {
return stripos(strrev($file), 'gpj.') === 0;
});

// Reduce the set for testing
// $files = $files->slice( 0, 200 );

// Place to target specific files for debug:
// $files = [
// 'images/3d473396-9994-c487-51a3-87c23132f0e5.jpg',
// 'images/9eabaa4f-bfcd-0fc0-1ea2-dbc64a8b0761.jpg',
// ];

// Grab just the ids - assumes that the folder is `images`
$ids = $files->map( function( $file ) {
return substr( $file, 7, strlen( $file ) - 11 );
});

$this->info( $ids->count() . ' files found...' );

if( !$this->option('force') ) {

$processed = Image::select('lake_guid')->whereNotNull('metadata->color')->get()->pluck('lake_guid');

$this->info( $processed->count() . ' images have already been processed...' );

$ids = $ids->diff( $processed );

}

if( $this->argument('start') !== null && $this->argument('count') !== null && $this->option('force') ) {

$this->info('Applying start of ' . $this->argument('start') . ' and count of ' . $this->argument('count') );

$ids = $ids->slice( (int) $this->argument('start'), (int) $this->argument('count') );

}

$this->info( $ids->count() . ' files will be processed.' );

$total = count( $ids );

foreach( $ids as $i => $id )
{

$file = 'images/' . $id . '.jpg';

// Skip touched files
if( Storage::size( $file ) < 1 ) {
$this->warn( $this->prefix( $i, $total, $id ) . 'Skipping touched file!' );
continue;
}

$contents = Storage::get( $file );

$manager = new ImageManager(array('driver' => 'imagick'));
$image = $manager->make( $contents );

try {
$palette = Palette::generate( $image );
} catch( \Exception $e ) {
// TODO: Resolve [ErrorException] max(): Array must contain at least one element
// See vendor/marijnvdwerf/material-palette/src/Palette.php:81
// https://github.com/marijnvdwerf/material-palette-php/issues/6
$this->warn( $this->prefix( $i, $total, $id ) . 'Monotone image skipped!' );
continue;
}

// TODO: Reorder these for better results?
$swatches = [
'vibrant' => $palette->getVibrantSwatch(),
'muted' => $palette->getMutedSwatch(),
'vibrant_light' => $palette->getLightVibrantSwatch(),
'muted_light' => $palette->getLightMutedSwatch(),
'vibrant_dark' => $palette->getDarkVibrantSwatch(),
'muted_dark' => $palette->getDarkMutedSwatch(),
];

$swatches = collect( $swatches );

// Select the first swatch that (1) isn't empty, and (2) isn't derived
$swatch = $swatches->first( function( $swatch ) {
return !is_null( $swatch ) && $swatch->getPopulation() > 0;
});

// I guess this might happen if the image is black-and-white?
if( !$swatch ) {
$this->warn( $this->prefix( $i, $total, $id ) . 'No swatches generated!' );
continue;
}

// Convert to HSL - better for searching w/ Elasticsearch:
// https://dpb587.me/blog/2014/04/24/color-searching-with-elasticsearch.html
$color = $swatch->getColor()->asHSLColor();

// For calculating percentage of pixel population
$size = getimagesize( storage_path() . '/app/' . $file );

// @TODO Consider using HSV instead
$out = [
'population' => $swatch->getPopulation(),
'percentage' => $swatch->getPopulation() / ( $size[0] * $size[1] ) * 100,
'h' => floor( $this->normalize( $color->getHue() * 360, 0, 360 ) ),
's' => floor( $this->normalize( $color->getSaturation() * 100, 0, 100 ) ),
'l' => floor( $this->normalize( $color->getLightness() * 100, 0, 100 ) ),
];

// Save the color to Image metadata
$image = Image::find( $id );

// Image not found
if( !$image ) {
$this->warn( $this->prefix( $i, $total, $id ) . 'Image not found!' );
continue;
}

$metadata = $image->metadata ?? (object) [];
$metadata->color = $out;

$image->metadata = $metadata;
$image->save();

$this->info( $this->prefix( $i, $total, $id ) . json_encode( $out ) );

}

}

/**
* Normalizes any number to an arbitrary range by assuming the range
* wraps around when going below min or above max.
*
* @link https://stackoverflow.com/questions/1628386/normalise-orientation-between-0-and-360
*/
private function normalize( $value, $min, $max )
{
$range = $max - $min;
$offset = $value - $min;

// + start to reset back to start of original range
return ( $offset - ( floor( $offset / $range ) * $range ) ) + $min;
}

/**
* Helper for consistent console output.
*/
private function prefix( $i, $total, $id )
{
return $i . ' of ' . $total . ': ' . $id . ' = ';
}

}
Loading

0 comments on commit 0567f65

Please sign in to comment.