From cb5186962ea3c0f459825f22c5d7c256192650e0 Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 25 Oct 2018 15:39:57 +0200 Subject: [PATCH 1/2] Support for CKAN 2.9 Use objects from toolkit, remove old checks using missing c properties, Add legacy routes --- ckanext/harvest/helpers.py | 4 +++- ckanext/harvest/plugin.py | 20 ++++++++++++++---- .../harvest/templates/source/admin_base.html | 21 +++++++++---------- .../harvest/templates/source/read_base.html | 13 +++++------- 4 files changed, 34 insertions(+), 24 deletions(-) diff --git a/ckanext/harvest/helpers.py b/ckanext/harvest/helpers.py index c43ecfd77..d00a27ca9 100644 --- a/ckanext/harvest/helpers.py +++ b/ckanext/harvest/helpers.py @@ -1,5 +1,4 @@ -from pylons import request from ckan import logic from ckan import model import ckan.lib.helpers as h @@ -9,6 +8,9 @@ from ckanext.harvest.plugin import DATASET_TYPE_NAME from ckanext.harvest.interfaces import IHarvester +request = p.toolkit.request + + def package_list_for_source(source_id): ''' Creates a dataset list with the ones belonging to a particular harvest diff --git a/ckanext/harvest/plugin.py b/ckanext/harvest/plugin.py index a1971445f..226142257 100644 --- a/ckanext/harvest/plugin.py +++ b/ckanext/harvest/plugin.py @@ -1,6 +1,7 @@ -import types +import json from logging import getLogger +from six import string_types from sqlalchemy.util import OrderedDict from ckan import logic @@ -175,12 +176,11 @@ def edit_template(self): return 'source/edit.html' def setup_template_variables(self, context, data_dict): - - p.toolkit.c.harvest_source = p.toolkit.c.pkg_dict + if hasattr(p.toolkit.c, 'pkg'): + p.toolkit.c.harvest_source = p.toolkit.c.pkg p.toolkit.c.dataset_type = DATASET_TYPE_NAME - def create_package_schema(self): ''' Returns the schema for mapping package data from a form to a format @@ -263,6 +263,18 @@ def update_config(self, config): p.toolkit.add_resource('fanstatic_library', 'ckanext-harvest') p.toolkit.add_resource('public/ckanext/harvest/javascript', 'harvest-extra-field') + if p.toolkit.check_ckan_version(min_version='2.9.0'): + mappings = config.get('ckan.legacy_route_mappings', {}) + if isinstance(mappings, string_types): + mappings = json.loads(mappings) + + mappings.update({ + 'harvest_read': 'harvest.read', + 'harvest_edit': 'harvest.edit', + }) + # https://github.com/ckan/ckan/pull/4521 + config['ckan.legacy_route_mappings'] = json.dumps(mappings) + ## IActions def get_actions(self): diff --git a/ckanext/harvest/templates/source/admin_base.html b/ckanext/harvest/templates/source/admin_base.html index 733428150..e6ebba5f2 100644 --- a/ckanext/harvest/templates/source/admin_base.html +++ b/ckanext/harvest/templates/source/admin_base.html @@ -8,42 +8,41 @@ {% endblock %} {% block action_links %} - {% set show_li = c.__version__.startswith('2.0') %} {% if source.status and source.status.last_job and (source.status.last_job.status == 'New' or source.status.last_job.status == 'Running') %} - {{ '
  • ' if show_li }} Reharvest{{ '
    • ' if show_li }} + Reharvest {% else %} {% set locale = h.dump_json({'content': _('This will re-run the harvesting for this source. Any updates at the source will overwrite the local datasets. Sources with a large number of datasets may take a significant amount of time to finish harvesting. Please confirm you would like us to start reharvesting.')}) %} - {{ '
  • ' if show_li }} + {{ _('Reharvest') }} - {{ '
    • ' if show_li }} + {% endif %} {% if source.status and source.status.last_job and (source.status.last_job.status == 'Running') %} - {{ '
  • ' if show_li }} + {{ _('Stop') }} - {{ '
    • ' if show_li }} + {% endif %} {% set locale = h.dump_json({'content': _('Warning: This will remove all datasets for this source, as well as all previous job reports. Are you sure you want to continue?')}) %} - {{ '
  • ' if show_li }} + {{ _('Clear') }} - {{ '
    • ' if show_li }} - {{ '
  • ' if show_li }} + + {{ _('View harvest source') }} - - {{ '
    • ' if show_li }} + + {% endblock %} {# CKAN 2.0 #} diff --git a/ckanext/harvest/templates/source/read_base.html b/ckanext/harvest/templates/source/read_base.html index 07ac75274..8f9867745 100644 --- a/ckanext/harvest/templates/source/read_base.html +++ b/ckanext/harvest/templates/source/read_base.html @@ -38,14 +38,11 @@

    {{ c.harvest_source.title }}

    {% block page_header %}
    - {# CKAN 2.1+ #} - {% if authorized_user and not c.__version__.startswith('2.0') %} - {% block content_action %} -
    - {{ self.admin_link() }} -
    - {% endblock %} - {% endif %} + {% block content_action %} +
    + {{ self.admin_link() }} +
    + {% endblock %}
      {% block page_header_tabs %} {{ h.build_nav_icon('{0}_read'.format(c.dataset_type), _('Datasets'), id=source.name, icon='sitemap') }} From dea0bd09ad774752b12908b7184613e79a11f17a Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 25 Oct 2018 15:48:42 +0200 Subject: [PATCH 2/2] Remove unused controller --- ckanext/harvest/controllers/organization.py | 201 -------------------- ckanext/harvest/plugin.py | 3 - 2 files changed, 204 deletions(-) delete mode 100644 ckanext/harvest/controllers/organization.py diff --git a/ckanext/harvest/controllers/organization.py b/ckanext/harvest/controllers/organization.py deleted file mode 100644 index 0b5b6ad07..000000000 --- a/ckanext/harvest/controllers/organization.py +++ /dev/null @@ -1,201 +0,0 @@ -import logging -from urllib import urlencode - -from ckan import plugins as p -from ckan.lib.base import c, model, request, render, h -from ckan.lib.base import abort -import ckan.lib.maintain as maintain -import ckan.lib.search as search - -from ckan.controllers.group import GroupController - -from ckanext.harvest.plugin import DATASET_TYPE_NAME - - -try: - from collections import OrderedDict # 2.7 -except ImportError: - from sqlalchemy.util import OrderedDict - -log = logging.getLogger(__name__) - -class OrganizationController(GroupController): - - def source_list(self, id, limit=20): - self.group_type = 'organization' - context = {'model': model, 'session': model.Session, - 'user': c.user or c.author, - 'schema': self._db_to_form_schema(group_type=self.group_type), - 'for_view': True} - data_dict = {'id': id} - - # unicode format (decoded from utf8) - q = c.q = request.params.get('q', '') - - try: - c.group_dict = self._action('organization_show')(context, data_dict) - c.group = context['group'] - except p.toolkit.ObjectNotFound: - abort(404, p.toolkit._('Group not found')) - except p.toolkit.NotAuthorized: - abort(401, p.toolkit._('Unauthorized to read group %s') % id) - - self._read(id, limit, dataset_type=DATASET_TYPE_NAME) - return render('source/org_source_list.html') - - def _read(self, id, limit, dataset_type=None): - ''' This is common code used by both read and bulk_process''' - self.group_type = 'organization' - context = {'model': model, 'session': model.Session, - 'user': c.user or c.author, - 'schema': self._db_to_form_schema(group_type=self.group_type), - 'for_view': True, 'extras_as_string': True} - - q = c.q = request.params.get('q', '') - # Search within group - if c.group_dict.get('is_organization'): - q += ' owner_org: "%s"' % c.group_dict.get('id') - else: - q += ' groups: "%s"' % c.group_dict.get('name') - - context['return_query'] = True - - try: - page = int(request.params.get('page', 1)) - except ValueError, e: - abort(400, ('"page" parameter must be an integer')) - - # most search operations should reset the page counter: - params_nopage = [(k, v) for k, v in request.params.items() - if k != 'page'] - #sort_by = request.params.get('sort', 'name asc') - sort_by = request.params.get('sort', None) - - def search_url(params): - if self.group_type == 'organization': - if c.action == 'bulk_process': - url = self._url_for(controller='organization', - action='bulk_process', - id=id) - elif c.action == 'source_list': - url = self._url_for('harvest_org_list', - id=id) - else: - url = self._url_for(controller='organization', - action='read', - id=id) - else: - url = self._url_for(controller='group', action='read', - id=id) - params = [(k, v.encode('utf-8') if isinstance(v, basestring) - else str(v)) for k, v in params] - return url + u'?' + urlencode(params) - - def drill_down_url(**by): - return h.add_url_param(alternative_url=None, - controller='group', action='read', - extras=dict(id=c.group_dict.get('name')), - new_params=by) - - c.drill_down_url = drill_down_url - - def remove_field(key, value=None, replace=None): - return h.remove_url_param(key, value=value, replace=replace, - controller='group', action='read', - extras=dict(id=c.group_dict.get('name'))) - - c.remove_field = remove_field - - def pager_url(q=None, page=None): - params = list(params_nopage) - params.append(('page', page)) - return search_url(params) - - try: - c.fields = [] - search_extras = {} - for (param, value) in request.params.items(): - if not param in ['q', 'page', 'sort'] \ - and len(value) and not param.startswith('_'): - if not param.startswith('ext_'): - c.fields.append((param, value)) - q += ' %s: "%s"' % (param, value) - else: - search_extras[param] = value - - fq = 'capacity:"public"' - user_member_of_orgs = [org['id'] for org - in h.organizations_available('read')] - - if (c.group and c.group.id in user_member_of_orgs): - fq = '' - context['ignore_capacity_check'] = True - - facets = OrderedDict() - - default_facet_titles = {'groups': p.toolkit._('Groups'), - 'tags': p.toolkit._('Tags'), - 'res_format': p.toolkit._('Formats'), - 'license': p.toolkit._('Licence'), } - - for facet in facets: - if facet in default_facet_titles: - facets[facet] = default_facet_titles[facet] - else: - facets[facet] = facet - if dataset_type: - fq = fq + 'dataset_type:"{dataset_type}"'.format(dataset_type=dataset_type) - - # Facet titles - for plugin in p.PluginImplementations(p.IFacets): - if self.group_type == 'organization': - facets = plugin.organization_facets( - facets, self.group_type, dataset_type) - else: - facets = plugin.group_facets( - facets, self.group_type, dataset_type) - - if 'capacity' in facets and (self.group_type != 'organization' or not user_member_of_orgs): - del facets['capacity'] - - c.facet_titles = facets - - data_dict = { - 'q': q, - 'fq': fq, - 'facet.field': facets.keys(), - 'rows': limit, - 'sort': sort_by, - 'start': (page - 1) * limit, - 'extras': search_extras - } - - query = p.toolkit.get_action('package_search')(context, data_dict) - - c.page = h.Page( - collection=query['results'], - page=page, - url=pager_url, - item_count=query['count'], - items_per_page=limit - ) - - c.facets = query['facets'] - maintain.deprecate_context_item( - 'facets', - 'Use `c.search_facets` instead.') - - c.search_facets = query['search_facets'] - c.search_facets_limits = {} - for facet in c.facets.keys(): - limit = int(request.params.get('_%s_limit' % facet, 10)) - c.search_facets_limits[facet] = limit - c.page.items = query['results'] - - c.sort_by_selected = sort_by - - except search.SearchError, se: - log.error('Group search error: %r', se.args) - c.query_error = True - c.facets = {} - c.page = h.Page(collection=[]) diff --git a/ckanext/harvest/plugin.py b/ckanext/harvest/plugin.py index 226142257..e7cb4bc18 100644 --- a/ckanext/harvest/plugin.py +++ b/ckanext/harvest/plugin.py @@ -245,9 +245,6 @@ def before_map(self, map): map.connect('harvest_object_show', '/' + DATASET_TYPE_NAME + '/object/:id', controller=controller, action='show_object') map.connect('harvest_object_for_dataset_show', '/dataset/harvest_object/:id', controller=controller, action='show_object', ref_type='dataset') - org_controller = 'ckanext.harvest.controllers.organization:OrganizationController' - map.connect('{0}_org_list'.format(DATASET_TYPE_NAME), '/organization/' + DATASET_TYPE_NAME + '/' + '{id}', controller=org_controller, action='source_list') - return map def update_config(self, config):