Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add HTTP Last-Modified header for TEI resources #396

Merged
merged 8 commits into from
Feb 8, 2022
Merged

Add HTTP Last-Modified header for TEI resources #396

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
c97ced4
Add HTTP Last-Modified header for TEI resources
joewiz Jan 21, 2022
e789d8d
Ensure Last-Modified is GMT (+0000)
joewiz Jan 21, 2022
23b4759
Work around header blindness outside controller
joewiz Jan 21, 2022
59008ca
Set 304 based on "If-Modified-Since" header
joewiz Jan 21, 2022
47f5b85
Clients request previously received Last-Modified
joewiz Jan 21, 2022
7f3a694
Truncate milliseconds from last-modified
joewiz Feb 3, 2022
8695ad3
Add HTTP Created header for TEI publications
joewiz Feb 3, 2022
043c955
Ensure last modified is present before truncating
joewiz Feb 4, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions controller.xql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,13 @@ console:log('exist:path: ' || $exist:path)
,
:)

let $if-modified-since := request:get-header("If-Modified-Since")
return
if ($if-modified-since) then
request:set-attribute("if-modified-since", $if-modified-since)
else
(),

if ($exist:path eq '') then
<dispatch xmlns="http://exist.sourceforge.net/NS/exist">
<redirect url="{local:get-uri()}/"/>
Expand Down
4 changes: 4 additions & 0 deletions modules/app.xqm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,10 @@ function app:handle-error($node as node(), $model as map(*), $code as xs:int?) {
()
};

declare function app:set-last-modified($last-modified as xs:dateTime) {
response:set-header("Last-Modified", format-dateTime(adjust-dateTime-to-timezone($last-modified, xs:dayTimeDuration("PT0H")), "[FNn,*-3], [D01] [MNn,*-3] [Y0001] [H01]:[m01]:[s01] [Z0000]", "en", (), ()))
};

declare function app:uri($node as node(), $model as map(*)) {
<code>{request:get-attribute("hsg-shell.path")}</code>
};
Expand Down
45 changes: 45 additions & 0 deletions modules/config.xqm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,19 @@ declare variable $config:PUBLICATIONS :=
map {
"frus": map {
"collection": $config:FRUS_VOLUMES_COL,
"document-last-modified": function($document-id) {
(
xmldb:last-modified($config:FRUS_VOLUMES_COL, $document-id || '.xml'),
(: for volumes that we do not have as TEI yet, fall back on volume metadata :)
xmldb:last-modified($config:FRUS_METADATA_COL, $document-id || '.xml')
)[1]
},
"section-last-modified": function($document-id, $section-id) {
(
xmldb:last-modified($config:FRUS_VOLUMES_COL, $document-id || '.xml'),
xmldb:last-modified($config:FRUS_METADATA_COL, $document-id || '.xml')
)[1]
},
"select-document": function($document-id) { doc($config:FRUS_VOLUMES_COL || '/' || $document-id || '.xml') },
"select-section": function($document-id, $section-id) {
let $node := doc($config:FRUS_VOLUMES_COL || '/' || $document-id || '.xml')/id($section-id)
Expand Down Expand Up @@ -155,6 +168,8 @@ declare variable $config:PUBLICATIONS :=
},
"buildings": map {
"collection": $config:BUILDINGS_COL,
"document-last-modified": function($document-id) { xmldb:last-modified($config:BUILDINGS_COL, $document-id || '.xml') },
"section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:BUILDINGS_COL, $document-id || '.xml') },
"select-document": function($document-id) { doc($config:BUILDINGS_COL || '/' || $document-id || '.xml') },
"select-section": function($document-id, $section-id) { doc($config:BUILDINGS_COL || '/' || $document-id || '.xml')/id($section-id) },
"html-href": function($document-id, $section-id) { "$app/departmenthistory/" || string-join(($document-id, $section-id), '/') },
Expand All @@ -171,6 +186,8 @@ declare variable $config:PUBLICATIONS :=
},
"conferences": map {
"collection": $config:CONFERENCES_ARTICLES_COL,
"document-last-modified": function($document-id) { xmldb:last-modified($config:CONFERENCES_ARTICLES_COL, $document-id || '.xml') },
"section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:CONFERENCES_ARTICLES_COL, $document-id || '.xml') },
"select-document": function($document-id) { doc($config:CONFERENCES_ARTICLES_COL || '/' || $document-id || '.xml') },
"select-section": function($document-id, $section-id) { doc($config:CONFERENCES_ARTICLES_COL || '/' || $document-id || '.xml')/id($section-id) },
"html-href": function($document-id, $section-id) { "$app/conferences/" || string-join(($document-id, $section-id), '/') },
Expand All @@ -193,6 +210,8 @@ declare variable $config:PUBLICATIONS :=
},
"countries": map {
"collection": $config:COUNTRIES_ARTICLES_COL,
"document-last-modified": function($document-id) { xmldb:last-modified($config:COUNTRIES_ARTICLES_COL, $document-id || '.xml') },
"section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:COUNTRIES_ARTICLES_COL, $document-id || '.xml') },
"select-document": function($document-id) { doc($config:COUNTRIES_ARTICLES_COL || '/' || $document-id || '.xml') },
"select-section": function($document-id, $section-id) { doc($config:COUNTRIES_ARTICLES_COL || '/' || $document-id || '.xml')/id($section-id) },
"html-href": function($document-id, $section-id) { "$app/countries/" || string-join(($document-id, $section-id), '/') },
Expand All @@ -203,6 +222,8 @@ declare variable $config:PUBLICATIONS :=
},
"countries-issues": map {
"collection": $config:COUNTRIES_ISSUES_COL,
"document-last-modified": function($document-id) { xmldb:last-modified($config:COUNTRIES_ISSUES_COL, $document-id || '.xml') },
"section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:COUNTRIES_ISSUES_COL, $document-id || '.xml') },
"select-document": function($document-id) { doc($config:COUNTRIES_ISSUES_COL || '/' || $document-id || '.xml') },
"select-section": function($document-id, $section-id) { doc($config:COUNTRIES_ISSUES_COL || '/' || $document-id || '.xml')/id($section-id) },
"html-href": function($document-id, $section-id) { "$app/countries/issues/" || string-join(($document-id, $section-id), '/') },
Expand All @@ -213,6 +234,8 @@ declare variable $config:PUBLICATIONS :=
},
"archives": map {
"collection": $config:ARCHIVES_ARTICLES_COL,
"document-last-modified": function($document-id) { xmldb:last-modified($config:ARCHIVES_ARTICLES_COL, $document-id || '.xml') },
"section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:ARCHIVES_ARTICLES_COL, $document-id || '.xml') },
"select-document": function($document-id) { doc($config:ARCHIVES_ARTICLES_COL || '/' || $document-id || '.xml') },
"select-section": function($document-id, $section-id) { doc($config:ARCHIVES_ARTICLES_COL || '/' || $document-id || '.xml')//tei:body },
"html-href": function($document-id, $section-id) { "$app/countries/" || string-join(($document-id, $section-id), '/') },
Expand All @@ -222,6 +245,8 @@ declare variable $config:PUBLICATIONS :=
},
"articles": map {
"collection": $config:FRUS_HISTORY_ARTICLES_COL,
"document-last-modified": function($document-id) { xmldb:last-modified($config:FRUS_HISTORY_ARTICLES_COL, $document-id || '.xml') },
"section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:FRUS_HISTORY_ARTICLES_COL, $document-id || '.xml') },
"select-document": function($document-id) { doc($config:FRUS_HISTORY_ARTICLES_COL || '/' || $document-id || '.xml') },
"select-section": function($document-id, $section-id) { doc($config:FRUS_HISTORY_ARTICLES_COL || '/' || $document-id || '.xml')//tei:body },
"html-href": function($document-id, $section-id) { "$app/frus-history/" || string-join(($document-id, $section-id), '/') },
Expand All @@ -240,6 +265,8 @@ declare variable $config:PUBLICATIONS :=
},
"people": map {
"collection": $config:SECRETARY_BIOS_COL,
"document-last-modified": function($document-id) { xmldb:last-modified($config:SECRETARY_BIOS_COL, $document-id || '.xml') },
"section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:SECRETARY_BIOS_COL, $document-id || '.xml') },
"select-document": function($document-id) { doc($config:SECRETARY_BIOS_COL || '/' || $document-id || '.xml') },
"select-section": function($document-id, $section-id) { doc($config:SECRETARY_BIOS_COL || '/' || $document-id || '.xml')/id($section-id) },
"html-href": function($document-id, $section-id) { "$app/departmenthistory/people/" || string-join(($document-id, $section-id), '/') },
Expand Down Expand Up @@ -285,6 +312,8 @@ declare variable $config:PUBLICATIONS :=
},
"milestones": map {
"collection": $config:MILESTONES_COL,
"document-last-modified": function($document-id) { xmldb:last-modified($config:MILESTONES_COL, $document-id || '.xml') },
"section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:MILESTONES_COL, $document-id || '.xml') },
"select-document": function($document-id) { doc($config:MILESTONES_COL || '/' || $document-id || '.xml') },
"select-section": function($document-id, $section-id) { doc($config:MILESTONES_COL|| '/' || $document-id || '.xml')/id($section-id) },
"html-href": function($document-id, $section-id) { "$app/milestones/" || string-join(($document-id, $section-id), '/') },
Expand All @@ -295,6 +324,8 @@ declare variable $config:PUBLICATIONS :=
},
"short-history": map {
"collection": $config:SHORT_HISTORY_COL,
"document-last-modified": function($document-id) { xmldb:last-modified($config:SHORT_HISTORY_COL, $document-id || '.xml') },
"section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:SHORT_HISTORY_COL, $document-id || '.xml') },
"select-document": function($document-id) { doc($config:SHORT_HISTORY_COL || '/' || $document-id || '.xml') },
"select-section": function($document-id, $section-id) { doc($config:SHORT_HISTORY_COL || '/' || $document-id || '.xml')/id($section-id) },
"html-href": function($document-id, $section-id) { "$app/departmenthistory/" || string-join(($document-id, $section-id), '/') },
Expand All @@ -305,6 +336,8 @@ declare variable $config:PUBLICATIONS :=
},
"timeline": map {
"collection": $config:ADMINISTRATIVE_TIMELINE_COL,
"document-last-modified": function($document-id) { xmldb:last-modified($config:ADMINISTRATIVE_TIMELINE_COL, $document-id || '.xml') },
"section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:ADMINISTRATIVE_TIMELINE_COL, $document-id || '.xml') },
"select-document": function($document-id) { doc($config:ADMINISTRATIVE_TIMELINE_COL || '/' || $document-id || '.xml') },
"select-section": function($document-id, $section-id) { doc($config:ADMINISTRATIVE_TIMELINE_COL || '/' || $document-id || '.xml')/id('chapter_' || $section-id) },
"html-href": function($document-id, $section-id) { "$app/departmenthistory/" || string-join(($document-id, substring-after($section-id, 'chapter_')), '/') },
Expand All @@ -316,6 +349,8 @@ declare variable $config:PUBLICATIONS :=
},
"faq": map {
"collection": $config:FAQ_COL,
"document-last-modified": function($document-id) { xmldb:last-modified($config:FAQ_COL, $document-id || '.xml') },
"section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:FAQ_COL, $document-id || '.xml') },
"select-document": function($document-id) { doc($config:FAQ_COL || '/' || $document-id || '.xml') },
"select-section": function($document-id, $section-id) { doc($config:FAQ_COL || '/' || $document-id || '.xml')/id($section-id) },
"html-href": function($document-id, $section-id) { "$app/about/" || string-join(($document-id, $section-id), '/') },
Expand All @@ -325,6 +360,8 @@ declare variable $config:PUBLICATIONS :=
},
"hac": map {
"collection": $config:HAC_COL,
"document-last-modified": function($document-id) { xmldb:last-modified($config:HAC_COL, $document-id || '.xml') },
"section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:HAC_COL, $document-id || '.xml') },
"select-document": function($document-id) { doc($config:HAC_COL || '/' || $document-id || '.xml') },
"select-section": function($document-id, $section-id) { doc($config:HAC_COL || '/' || $document-id || '.xml')/id($section-id) },
"html-href": function($document-id, $section-id) { "$app/about/" || string-join(($document-id, $section-id), '/') },
Expand All @@ -334,6 +371,8 @@ declare variable $config:PUBLICATIONS :=
},
"education": map {
"collection": $config:EDUCATION_COL,
"document-last-modified": function($document-id) { xmldb:last-modified($config:EDUCATION_COL, $document-id || '.xml') },
"section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:EDUCATION_COL, $document-id || '.xml') },
"select-document": function($document-id) { doc($config:EDUCATION_COL || '/' || $document-id || '.xml') },
"select-section": function($document-id, $section-id) { doc($config:EDUCATION_COL || '/' || $document-id || '.xml')/id($section-id) },
"html-href": function($document-id, $section-id) { "$app/education/modules/" || string-join(($document-id, $section-id), '#') },
Expand All @@ -346,6 +385,8 @@ declare variable $config:PUBLICATIONS :=
},
"frus-history-monograph": map {
"collection": $config:FRUS_HISTORY_MONOGRAPH_COL,
"document-last-modified": function($document-id) { xmldb:last-modified($config:FRUS_HISTORY_MONOGRAPH_COL, $document-id || '.xml') },
"section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:FRUS_HISTORY_MONOGRAPH_COL, $document-id || '.xml') },
"select-document": function($document-id) { doc($config:FRUS_HISTORY_MONOGRAPH_COL || '/' || $document-id || '.xml') },
"select-section": function($document-id, $section-id) {
let $target-section-id :=
Expand Down Expand Up @@ -392,6 +433,8 @@ declare variable $config:PUBLICATIONS :=
},
"vietnam-guide": map {
"collection": $config:VIETNAM_GUIDE_COL,
"document-last-modified": function($document-id) { xmldb:last-modified($config:VIETNAM_GUIDE_COL, $document-id || '.xml') },
"section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:VIETNAM_GUIDE_COL, $document-id || '.xml') },
"select-document": function($document-id) { doc($config:VIETNAM_GUIDE_COL || '/' || $document-id || '.xml') },
"select-section": function($document-id, $section-id) { doc($config:VIETNAM_GUIDE_COL || '/' || $document-id || '.xml') },
"html-href": function($document-id, $section-id) { "$app/historicaldocuments/" || string-join(($document-id, $section-id), '/') },
Expand All @@ -401,6 +444,8 @@ declare variable $config:PUBLICATIONS :=
},
"views-from-the-embassy": map {
"collection": $config:VIEWS_FROM_EMBASSY_COL,
"document-last-modified": function($document-id) { xmldb:last-modified($config:VIEWS_FROM_EMBASSY_COL, $document-id || '.xml') },
"section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:VIEWS_FROM_EMBASSY_COL, $document-id || '.xml') },
"select-document": function($document-id) { doc($config:VIEWS_FROM_EMBASSY_COL || '/' || $document-id || '.xml') },
"select-section": function($document-id, $section-id) { doc($config:VIEWS_FROM_EMBASSY_COL || '/' || $document-id || '.xml')/id($section-id) },
"html-href": function($document-id, $section-id) { "$app/departmenthistory/wwi" },
Expand Down
71 changes: 53 additions & 18 deletions modules/pages.xqm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,25 +44,60 @@ function pages:load($node as node(), $model as map(*), $publication-id as xs:str
$section-id as xs:string?, $view as xs:string, $ignore as xs:boolean) {
let $log := console:log("loading publication-id: " || $publication-id || " document-id: " || $document-id || " section-id: " || $section-id )

let $content := map {
"data":
if (exists($publication-id) and exists($document-id)) then
pages:load-xml($publication-id, $document-id, $section-id, $view, $ignore)
else (),
"publication-id": $publication-id,
"document-id": $document-id,
"section-id": $section-id,
"view": $view,
"base-path":
(: allow for pages that don't have $config:PUBLICATIONS?select-document defined :)
if (exists($publication-id) and map:contains(map:get($config:PUBLICATIONS, $publication-id), 'base-path')) then
map:get($config:PUBLICATIONS, $publication-id)?base-path($document-id, $section-id)
else (),
"odd": if (exists($publication-id)) then map:get($config:PUBLICATIONS, $publication-id)?transform else $config:odd-transform-default
}

let $last-modified :=
if (exists($publication-id) and exists($document-id)) then
pages:last-modified($publication-id, $document-id, $section-id)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
pages:last-modified($publication-id, $document-id, $section-id)
pages:last-modified($publication-id, $document-id, $section-id) => format-dateTime("[FNn,*-3], [D01] [MNn,*-3] [Y0001] [H01]:[m01]:[s01] [Z0000]", "en", (), ()) => parse-ietf-date()

xmldb:last-modified() has millisecond resolution.
HTTP Last-Modified and If-Modified-Since headers have second resolution.
We need to make sure we mangle this the same way for outgoing and incoming HTTP headers and xmldb:last-modified() calls so comparisons work as expected.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@agh2342 I've applied the change and believe it correctly truncates milliseconds. To confirm the approach:

xquery version "3.1";

let $date := current-dateTime()
return
    (
        $date,
        $date
        => format-dateTime("[Y0001]-[M01]-[D01]T[H01]:[m01]:[s01][Z]")
        => xs:dateTime()
    )

This returns:

(
    xs:dateTime("2022-02-03T12:30:10.395-05:00"),
    xs:dateTime("2022-02-03T12:30:10-05:00")
)

I deployed the change locally, and the calculation appears to be correct.

else
()
let $if-modified-since := try { request:get-attribute("if-modified-since") => parse-ietf-date() } catch * { () }
let $should-return-304 :=
if (exists($last-modified) and exists($if-modified-since)) then
$if-modified-since ge $last-modified
else
()
return
templates:process($node/*, map:merge(($model, $content)))
(: if the "If-Modified-Since" header in the client request is later than the
: last-modified date, then halt further processing of the templates and simply
: return a 304 response. :)
if ($should-return-304) then
(
response:set-status-code(304),
app:set-last-modified($last-modified)
)
else
let $content := map {
"data":
if (exists($publication-id) and exists($document-id)) then
pages:load-xml($publication-id, $document-id, $section-id, $view, $ignore)
else (),
"publication-id": $publication-id,
"document-id": $document-id,
"section-id": $section-id,
"view": $view,
"base-path":
(: allow for pages that do not have $config:PUBLICATIONS?select-document defined :)
(: ... TODO: I do not see any such cases in config:PUBLICATIONS! Check if OK to remove this entry? - JW :)
if (exists($publication-id) and map:contains(map:get($config:PUBLICATIONS, $publication-id), 'base-path')) then
map:get($config:PUBLICATIONS, $publication-id)?base-path($document-id, $section-id)
else (),
"odd": if (exists($publication-id)) then map:get($config:PUBLICATIONS, $publication-id)?transform else $config:odd-transform-default
}

return
(
if (exists($last-modified)) then
request:set-attribute("hsgshell.last-modified", $last-modified)
else
(),
templates:process($node/*, map:merge(($model, $content)))
)
};

declare function pages:last-modified($publication-id as xs:string, $document-id as xs:string, $section-id as xs:string?) {
if ($section-id) then
map:get($config:PUBLICATIONS, $publication-id)?section-last-modified($document-id, $section-id)
else
map:get($config:PUBLICATIONS, $publication-id)?document-last-modified($document-id)
};

declare function pages:load-xml($publication-id as xs:string, $document-id as xs:string, $section-id as xs:string?, $view as xs:string) {
Expand Down
Loading