From 8414432f785959fea37d69900b0b54caa046d2ba Mon Sep 17 00:00:00 2001 From: Rae Knowler Date: Mon, 15 Aug 2022 14:20:48 +0200 Subject: [PATCH 1/6] Use fq for dataset search, not q The Solr maxBooleanClauses config only applies to terms in a query string, not a filter query: https://solr.apache.org/guide/8_1/query-settings-in-solrconfig.html#maxbooleanclauses This will prevent Solr crashing when a showcase has >1024 datasets associated with it. When searching for datasets that are *not* associated wtih a showcase, we already use an equivalent query in the fq parameter: https://github.com/ckan/ckanext-showcase/blob/master/ckanext/showcase/utils.py#L313. --- ckanext/showcase/logic/action/get.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckanext/showcase/logic/action/get.py b/ckanext/showcase/logic/action/get.py index aa6b7de0..1d9a93d2 100644 --- a/ckanext/showcase/logic/action/get.py +++ b/ckanext/showcase/logic/action/get.py @@ -75,10 +75,10 @@ def showcase_package_list(context, data_dict): id_list = [] for pkg_id in pkg_id_list: id_list.append(pkg_id[0]) - q = 'id:(' + ' OR '.join(['{0}'.format(x) for x in id_list]) + ')' + fq = 'id:(' + ' OR '.join(['{0}'.format(x) for x in id_list]) + ')' _pkg_list = toolkit.get_action('package_search')( context, - {'q': q, 'rows': 100}) + {'fq': fq, 'rows': 100}) pkg_list = _pkg_list['results'] return pkg_list From 64e0603914cd68a01afa1bc3c9716333a215c89c Mon Sep 17 00:00:00 2001 From: Rae Knowler Date: Wed, 10 Aug 2022 18:02:45 +0200 Subject: [PATCH 2/6] Paginate list of packages associated with a showcase --- ckanext/showcase/logic/action/get.py | 19 +++++++++++---- ckanext/showcase/templates/showcase/read.html | 2 +- ckanext/showcase/utils.py | 23 +++++++++++++++++-- ckanext/showcase/views.py | 1 - 4 files changed, 37 insertions(+), 8 deletions(-) diff --git a/ckanext/showcase/logic/action/get.py b/ckanext/showcase/logic/action/get.py index 1d9a93d2..25e92cc9 100644 --- a/ckanext/showcase/logic/action/get.py +++ b/ckanext/showcase/logic/action/get.py @@ -46,10 +46,17 @@ def showcase_list(context, data_dict): @toolkit.side_effect_free def showcase_package_list(context, data_dict): - '''List packages associated with a showcase. + '''Get paginated list of packages associated with a showcase. :param showcase_id: id or name of the showcase :type showcase_id: string + :param limit: the list of datasets will be broken into pages of at most + ``limit`` datasets per page and only one page will be returned at a + time (optional, default: ckan.datasets_per_page or 20) + :type limit: int + :param offset: when ``limit`` is given, the offset to start + returning packages from + :type offset: int :rtype: list of dictionaries ''' @@ -64,16 +71,20 @@ def showcase_package_list(context, data_dict): if errors: raise toolkit.ValidationError(errors) + # Todo: Add separate config option for this instead of reusing this one. + limit = data_dict.get('limit', toolkit.config.get('ckan.datasets_per_page', 20)) + offset = data_dict.get('offset', 0) + # get a list of package ids associated with showcase id pkg_id_list = ShowcasePackageAssociation.get_package_ids_for_showcase( validated_data_dict['showcase_id']) pkg_list = [] if pkg_id_list: - # for each package id, get the package dict and append to list if - # active + # for each package id in the range given by limit and offset, + # get the package dict and append to list if active id_list = [] - for pkg_id in pkg_id_list: + for pkg_id in pkg_id_list[offset:offset + limit]: id_list.append(pkg_id[0]) fq = 'id:(' + ' OR '.join(['{0}'.format(x) for x in id_list]) + ')' _pkg_list = toolkit.get_action('package_search')( diff --git a/ckanext/showcase/templates/showcase/read.html b/ckanext/showcase/templates/showcase/read.html index 62fbf8ac..93768acc 100644 --- a/ckanext/showcase/templates/showcase/read.html +++ b/ckanext/showcase/templates/showcase/read.html @@ -107,7 +107,7 @@

{% block secondary_help_content %}{% endblock %} {% block package_info %} - {% snippet 'showcase/snippets/showcase_info.html', pkg=pkg, showcase_pkgs=c.showcase_pkgs %} + {% snippet 'showcase/snippets/showcase_info.html', pkg=pkg, showcase_pkgs=page.items %} {% endblock %} {% block package_social %} diff --git a/ckanext/showcase/utils.py b/ckanext/showcase/utils.py index f388bdee..38e1fc16 100644 --- a/ckanext/showcase/utils.py +++ b/ckanext/showcase/utils.py @@ -64,6 +64,7 @@ def check_new_view_auth(): def read_view(id): + page_number = h.get_page_number(tk.request.params) context = { 'model': model, 'session': model.Session, @@ -82,14 +83,32 @@ def read_view(id): return tk.abort(401, _('Unauthorized to read showcase')) # get showcase packages - tk.g.showcase_pkgs = tk.get_action('ckanext_showcase_package_list')( + num_datasets = tk.get_action('ckanext_showcase_package_list_count')( context, { 'showcase_id': tk.g.pkg_dict['id'] }) + limit = int(tk.config.get(u'ckan.datasets_per_page', 20)) + offset = (page_number - 1) * limit + + showcase_pkgs = tk.get_action('ckanext_showcase_package_list')( + context, { + 'showcase_id': tk.g.pkg_dict['id'], + 'limit': limit, + 'offset': offset + }) + + page = h.Page( + collection=showcase_pkgs, + page=page_number, + url=h.pager_url, + item_count=num_datasets, + items_per_page=limit, + presliced_list=True + ) package_type = DATASET_TYPE_NAME return tk.render('showcase/read.html', - extra_vars={'dataset_type': package_type}) + extra_vars={'dataset_type': package_type, 'page': page}) def manage_datasets_view(id): diff --git a/ckanext/showcase/views.py b/ckanext/showcase/views.py index f054085a..cb300fc4 100644 --- a/ckanext/showcase/views.py +++ b/ckanext/showcase/views.py @@ -2,7 +2,6 @@ from flask import Blueprint - import ckan.lib.helpers as h import ckan.plugins.toolkit as tk import ckan.views.dataset as dataset From ea70beca7461b521d0b1bda2705963d1920a415e Mon Sep 17 00:00:00 2001 From: Rae Knowler Date: Wed, 10 Aug 2022 18:04:17 +0200 Subject: [PATCH 3/6] Add action to get number of datasets in a showcase Necessary because the action ckanext_showcase_package_list now returns a paginated result --- ckanext/showcase/logic/action/__init__.py | 2 ++ ckanext/showcase/logic/action/get.py | 27 +++++++++++++++++++++++ ckanext/showcase/plugin/__init__.py | 5 ++--- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/ckanext/showcase/logic/action/__init__.py b/ckanext/showcase/logic/action/__init__.py index 43ab2d14..72a5e083 100644 --- a/ckanext/showcase/logic/action/__init__.py +++ b/ckanext/showcase/logic/action/__init__.py @@ -22,6 +22,8 @@ def get_actions(): ckanext.showcase.logic.action.delete.showcase_package_association_delete, 'ckanext_showcase_package_list': ckanext.showcase.logic.action.get.showcase_package_list, + 'ckanext_showcase_package_list_count': + ckanext.showcase.logic.action.get.showcase_package_list_count, 'ckanext_package_showcase_list': ckanext.showcase.logic.action.get.package_showcase_list, 'ckanext_showcase_admin_add': diff --git a/ckanext/showcase/logic/action/get.py b/ckanext/showcase/logic/action/get.py index 25e92cc9..14e92ae1 100644 --- a/ckanext/showcase/logic/action/get.py +++ b/ckanext/showcase/logic/action/get.py @@ -44,6 +44,33 @@ def showcase_list(context, data_dict): return showcase_list +@toolkit.side_effect_free +def showcase_package_list_count(context, data_dict): + '''Get number of packages associated with a showcase. + + :param showcase_id: id or name of the showcase + :type showcase_id: string + + :rtype: int + ''' + + toolkit.check_access('ckanext_showcase_package_list', context, data_dict) + + # validate the incoming data_dict + validated_data_dict, errors = validate(data_dict, + showcase_package_list_schema(), + context) + + if errors: + raise toolkit.ValidationError(errors) + + # get a list of package ids associated with showcase id + pkg_id_list = ShowcasePackageAssociation.get_package_ids_for_showcase( + validated_data_dict['showcase_id']) + + return len(pkg_id_list) + + @toolkit.side_effect_free def showcase_package_list(context, data_dict): '''Get paginated list of packages associated with a showcase. diff --git a/ckanext/showcase/plugin/__init__.py b/ckanext/showcase/plugin/__init__.py index ff36a0da..14708d9e 100644 --- a/ckanext/showcase/plugin/__init__.py +++ b/ckanext/showcase/plugin/__init__.py @@ -162,9 +162,8 @@ def _add_to_pkg_dict(self, context, pkg_dict): qualified=True) # Add dataset count - pkg_dict[u'num_datasets'] = len( - tk.get_action('ckanext_showcase_package_list')( - context, {'showcase_id': pkg_dict['id']})) + pkg_dict[u'num_datasets'] = tk.get_action('ckanext_showcase_package_list_count')( + context, {'showcase_id': pkg_dict['id']}) # Rendered notes if showcase_helpers.get_wysiwyg_editor() == 'ckeditor': From 7bc82f6df492b7a61f435b097067f1db50bec75e Mon Sep 17 00:00:00 2001 From: Rae Knowler Date: Thu, 11 Aug 2022 11:56:21 +0200 Subject: [PATCH 4/6] Convert value from config to int before using --- ckanext/showcase/logic/action/get.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ckanext/showcase/logic/action/get.py b/ckanext/showcase/logic/action/get.py index 14e92ae1..39ac1cd5 100644 --- a/ckanext/showcase/logic/action/get.py +++ b/ckanext/showcase/logic/action/get.py @@ -99,7 +99,8 @@ def showcase_package_list(context, data_dict): raise toolkit.ValidationError(errors) # Todo: Add separate config option for this instead of reusing this one. - limit = data_dict.get('limit', toolkit.config.get('ckan.datasets_per_page', 20)) + limit = data_dict.get( + 'limit', int(toolkit.config.get('ckan.datasets_per_page', 20))) offset = data_dict.get('offset', 0) # get a list of package ids associated with showcase id From edeed5e72a9bec1b2d1111110db86941d77b40d5 Mon Sep 17 00:00:00 2001 From: Rae Knowler Date: Mon, 15 Aug 2022 11:56:48 +0200 Subject: [PATCH 5/6] WIP: display showcase datasets paginated on manage-datasets page --- .../i18n/de/LC_MESSAGES/ckanext-showcase.mo | Bin 9304 -> 9304 bytes .../i18n/fr/LC_MESSAGES/ckanext-showcase.mo | Bin 9034 -> 9551 bytes .../LC_MESSAGES/ckanext-showcase.mo | Bin 8355 -> 8355 bytes .../templates/showcase/manage_datasets.html | 11 ++++- ckanext/showcase/utils.py | 43 +++++++++++++----- 5 files changed, 40 insertions(+), 14 deletions(-) diff --git a/ckanext/showcase/i18n/de/LC_MESSAGES/ckanext-showcase.mo b/ckanext/showcase/i18n/de/LC_MESSAGES/ckanext-showcase.mo index c77f1d2ee73e3269a46ee5a7f9f289f0469b4e81..9b695ee07f944ec3552ef459b432238f9b7096f1 100644 GIT binary patch delta 16 XcmccNal>Q7BN1j}J(JB(MT+?WKyL=E delta 16 XcmccNal>Q7BN1jxJ;TjUMT+?WK#c~l diff --git a/ckanext/showcase/i18n/fr/LC_MESSAGES/ckanext-showcase.mo b/ckanext/showcase/i18n/fr/LC_MESSAGES/ckanext-showcase.mo index 2d57081380d93ea85da4339cea45ad574d0374e7..4f6291750a4da2321d268a7eaecf1d446c13ed97 100644 GIT binary patch delta 2232 zcmZA1eQ4EH9LMo<_ndR?(YfdS?%bSfkEN@0wxTV~nfpf&tq`x?J9q8w{%rSmS4_4m zR7g_JxJ@vlf5u!ciR?l|8dO43BpKNQC?Od5$1FxANi*vG*?!X8Z;UatqQ*i{daTFKhIA-7nSb%5T=NDbC zVjJz#*n|yq-e643JUv%1V_3(7DNMtq_ZYJT%UtVG0oP(Kb|G__XWY6U8EOXbUW{Ru z$Cyb}LLZTRFa{xhtZ2K zqgFnF`FIwU$VDu{s~A(@848+U2~mo$2utxktisJm7qicG2sP0m+=*|Zc5Vi@qn7D| zr*H+HMGLRFrZBjcdQCR_uMV4OXh8>?a2zeXggTm=xCv)a_j5ze-1AMSoeJS{d=+(x z-^X&ifDAEHIGdo`t_xI0dqpn$Z&7%H2Ay3HS%m4wrFa0fbw`o0W)hd-8LY5B<0{-jbPYU+x(g$& z@1S0v&s;B~R(2B=AeGKa$cviie*7AHPzkPL@M`QvE#O7ue#gvfZo@Iu5saY*oI-8o zc~nA|T(2Ml%`ZsKW*QYZok1F3gnA3SsKi#fcA}1UC#ruKYxVxeC{)lefsOdZ>;q%| zM82ksSv2rLY{kb>3G7D&I*9sza2U0+W2lvmq2~Pw^%nhwx}+tXv+hni&c6SFKXUNj6Fu- z2MX#~Y_Uq*hPtIkP_N$@cH>FZ>vs#)?>4T()!w-kKZmNHz_oY{dF4$7%h!%IVgs&0 z?NCn@``3xm(^^6CX?)T-0K9`W#;*;6xICqaH5iM0G)Pi~8~HxRY_0n2ZP!{<&0B8k3&9Z75Ex0tTGdsxA+({D!ye7iyc zJ$qq(=fwHKb4khFb~yfTWm&r81hmHZWJ!7aQ1P>gsp6yQ3j-PwxO<8N-qW7?+2t+B M&0X}u0W7}sKa5~MeEI)&0`Svgz3oJA0gjBK6ms@wHr^G-KzU0>W< zqmnkc6;VdS2BVS+zEC5f&HiZ{tB8WgKVpscpRo0VF(RVp%lkqd=RH23&-e4={rP;} zUvD=4zRbBlA^WiL*}~tO{56eG?bT;@j#($w6pqBFn2*oUk9oOfqi`Y?VG#Aa#&any zpuP@6*n`XQmyC{$Vss@9YmhlDiQ}-#^IOz}C$Rv}<6Ag@V|`{rsD+=6HY>sWyv#h6 zIEH#PD)2hY#s(aV%ZGJ{i~_+}8!CVp^0jt;)G>t$=peF|9Ygi|9^b$-r~n60{cd^n zKe3ehzc>NM5KR!LpypkSA=bB56coU2&%MZItQ*Vm2x^BvVIf{eE%*Snz$4TGe(po# z0yqgPu?!n2X%5k;zaDnX?S})=dZ~Br$Gw@$S@AUs!(@ZhiYGoI_fsmQSZTO z?7=3yikdi|ho^m<3ieXJ#j3iJo!lQw*k`DD_M-y(!J(j?Uq}_4=%! zcO&k__wgpq!RbV)aT`!~-s<@|>TUSW^Bk)GulNT(M+M^iM%Z&HJVZrW!Yt&IRiNrs zs3V<^8qk0`p%&D_A9=Q;#wC%tZ69jKCs5i8p)3kkhwL?89!aeLfl6N&Q_+Vm~gzS~7SccHm-kQS&~*2G+L#VX9** z&cjovB)f-tZH90)K1RJZ?~pL+w-oCzh1&6Dul^6dO?_4{^ABk&>ZCex7Jh;1e;%C$ z6dqD2!eDvk?rKmwY(mzwR^(8u3v00#tME4Ju19dbyy8}hs#l^;v>r8X4QiekF2MsB z#v4;fA+E`)E7EznN%x!FJ3e>c=uflKGyE;?L4RPR+h4HGm!4g?p>Ws7k)0og6XE)J zv@H=0?+9&aPaiGpC<^Urk8bOTY=8ZMTVAxy=N>O^$#QEZ{hXDaR~B>o%DQvi=86lx cl6W)``5+NZY>%XZ@kBTgjVGd?bVTCz5+zQ@fB*mh diff --git a/ckanext/showcase/i18n/zh_Hant_TW/LC_MESSAGES/ckanext-showcase.mo b/ckanext/showcase/i18n/zh_Hant_TW/LC_MESSAGES/ckanext-showcase.mo index 2465e7d6a9423e06fdd99e4a4f633dd3bd0ab71a..3e5044a2952b3ec4c42ce97d819ef6725b115ed8 100644 GIT binary patch delta 16 YcmZ4NxY%*SPZ4HgJ(JCUM3!>{06Q`UlK=n! delta 16 YcmZ4NxY%*SPZ4HwJ%i1EM3!>{06RJclK=n! diff --git a/ckanext/showcase/templates/showcase/manage_datasets.html b/ckanext/showcase/templates/showcase/manage_datasets.html index b1da5c6e..cd87026e 100644 --- a/ckanext/showcase/templates/showcase/manage_datasets.html +++ b/ckanext/showcase/templates/showcase/manage_datasets.html @@ -94,7 +94,7 @@

{{ _('Datasets in this showcase') }}

- {% if c.showcase_pkgs %} + {% if pkg_page %}
@@ -113,7 +113,7 @@

{{ _('Datasets in this showcase') }}

- {% for package in c.showcase_pkgs %} + {% for package in pkg_page.items %} {% set truncate = truncate or 180 %} {% set truncate_title = truncate_title or 80 %} {% set title = package.title or package.name %} @@ -133,6 +133,13 @@

{% endfor %}

+ {% if pkg_page.pager() %} + + + + + + {% endif %}
{% else %} diff --git a/ckanext/showcase/utils.py b/ckanext/showcase/utils.py index 38e1fc16..6addbdd5 100644 --- a/ckanext/showcase/utils.py +++ b/ckanext/showcase/utils.py @@ -5,9 +5,11 @@ import logging from collections import OrderedDict +from pylons import url as _pylons_default_url import six from six.moves.urllib.parse import urlencode +from ckan.common import is_flask_request import ckan.model as model import ckan.plugins as p import ckan.logic as logic @@ -83,32 +85,49 @@ def read_view(id): return tk.abort(401, _('Unauthorized to read showcase')) # get showcase packages + page = _get_showcase_packages_page(context, page_number) + + package_type = DATASET_TYPE_NAME + return tk.render('showcase/read.html', + extra_vars={'dataset_type': package_type, 'page': page}) + + +def _get_showcase_packages_page(context, page_number): num_datasets = tk.get_action('ckanext_showcase_package_list_count')( context, { 'showcase_id': tk.g.pkg_dict['id'] }) limit = int(tk.config.get(u'ckan.datasets_per_page', 20)) offset = (page_number - 1) * limit - showcase_pkgs = tk.get_action('ckanext_showcase_package_list')( context, { 'showcase_id': tk.g.pkg_dict['id'], 'limit': limit, 'offset': offset }) - page = h.Page( collection=showcase_pkgs, page=page_number, - url=h.pager_url, + url=_pager_url, item_count=num_datasets, items_per_page=limit, presliced_list=True ) + return page - package_type = DATASET_TYPE_NAME - return tk.render('showcase/read.html', - extra_vars={'dataset_type': package_type, 'page': page}) + +def _pager_url(page, **kwargs): + pargs = [] + if is_flask_request(): + pargs.append(tk.request.endpoint) + else: + routes_dict = _pylons_default_url.environ['pylons.routes_dict'] + kwargs['controller'] = routes_dict['controller'] + kwargs['action'] = routes_dict['action'] + if routes_dict.get('id'): + kwargs['id'] = routes_dict['id'] + kwargs['page'] = page + return tk.url_for(*pargs, **kwargs) def manage_datasets_view(id): @@ -202,13 +221,13 @@ def manage_datasets_view(id): _add_dataset_search(tk.g.pkg_dict['id'], tk.g.pkg_dict['name']) - # get showcase packages - tk.g.showcase_pkgs = tk.get_action('ckanext_showcase_package_list')( - context, { - 'showcase_id': tk.g.pkg_dict['id'] - }) + # get showcase packages page + pkg_page_number = h.get_page_number(tk.request.params, key='pkg_page') + log.warn(pkg_page_number) + pkg_page = _get_showcase_packages_page(context, pkg_page_number) + log.warn(pkg_page) - return tk.render('showcase/manage_datasets.html') + return tk.render('showcase/manage_datasets.html', extra_vars={'pkg_page': pkg_page}) def _add_dataset_search(showcase_id, showcase_name): ''' From fcef19598e70037703e2860c37c4553428dc4d7a Mon Sep 17 00:00:00 2001 From: Rae Knowler Date: Tue, 16 Aug 2022 13:35:30 +0200 Subject: [PATCH 6/6] WIP figure out how to have two paginated lists on the same page --- ckanext/showcase/utils.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ckanext/showcase/utils.py b/ckanext/showcase/utils.py index 6addbdd5..9c5e8686 100644 --- a/ckanext/showcase/utils.py +++ b/ckanext/showcase/utils.py @@ -66,7 +66,7 @@ def check_new_view_auth(): def read_view(id): - page_number = h.get_page_number(tk.request.params) + page_number = h.get_page_number(tk.request.params, key='pkg_page') context = { 'model': model, 'session': model.Session, @@ -111,12 +111,14 @@ def _get_showcase_packages_page(context, page_number): url=_pager_url, item_count=num_datasets, items_per_page=limit, - presliced_list=True + presliced_list=True, + page_param='pkg_page' ) return page -def _pager_url(page, **kwargs): +def _pager_url(**kwargs): + log.warn(kwargs) pargs = [] if is_flask_request(): pargs.append(tk.request.endpoint) @@ -126,7 +128,8 @@ def _pager_url(page, **kwargs): kwargs['action'] = routes_dict['action'] if routes_dict.get('id'): kwargs['id'] = routes_dict['id'] - kwargs['page'] = page + page_param = kwargs['page_param'] + kwargs[page_param] = kwargs['page'] return tk.url_for(*pargs, **kwargs) @@ -223,9 +226,7 @@ def manage_datasets_view(id): # get showcase packages page pkg_page_number = h.get_page_number(tk.request.params, key='pkg_page') - log.warn(pkg_page_number) pkg_page = _get_showcase_packages_page(context, pkg_page_number) - log.warn(pkg_page) return tk.render('showcase/manage_datasets.html', extra_vars={'pkg_page': pkg_page})