Pulled up r14164 r14172 from trunk.

git-svn-id: https://svn.calendarserver.org/repository/calendarserver/CalendarServer/branches/release/CalendarServer-6.0-dev@14174 e27351fd-9f3e-4f54-a53b-843176b1656c

twistedcaldav/stdconfig.py

@@ -838,6 +838,17 @@
     # How many results to return for principal-property-search REPORT requests
     "MaxPrincipalSearchReportResults": 500,
 
+    #
+    # Client fixes per user-agent match
+    #
+    "ClientFixes" : {
+        "ForceAttendeeTRANSP" : [
+            "iOS/8\\.0(\\..*)?",
+            "iOS/8\\.1(\\..*)?",
+            "iOS/8\\.2(\\..*)?",
+        ],
+    },
+
     #
     # Localization
     #
@@ -1458,6 +1469,19 @@ def _updateRejectClients(configDict, reloading=False):
 
 
 
+def _updateClientFixes(configDict, reloading=False):
+    #
+    # Compile ClientFixes expressions for speed
+    #
+    try:
+        configDict.ClientFixesCompiled = {}
+        for key, expressions in configDict.ClientFixes.items():
+            configDict.ClientFixesCompiled[key] = [re.compile("^{}$".format(x)) for x in expressions]
+    except re.error, e:
+        raise ConfigurationError("Invalid regular expression in ClientFixes: %s" % (e,))
+
+
+
 def _updateLogLevels(configDict, reloading=False):
     log.publisher.levels.clearLogLevels()
 
@@ -1648,6 +1672,7 @@ def _updateCompliance(configDict, reloading=False):
     _postUpdateAugmentService,
     _updateACLs,
     _updateRejectClients,
+    _updateClientFixes,
     _updateLogLevels,
     _updateNotifications,
     _updateICalendar,

twistedcaldav/storebridge.py

@@ -49,7 +49,7 @@
 from twistedcaldav.sharing import (
     invitationBindStatusToXMLMap, invitationBindModeToXMLMap
 )
-from twistedcaldav.util import bestAcceptType
+from twistedcaldav.util import bestAcceptType, matchClientFixes
 from twistedcaldav.vcard import Component as VCard, InvalidVCardDataError
 from txdav.base.propertystore.base import PropertyName
 from txdav.caldav.icalendarstore import (
@@ -60,7 +60,7 @@
     InvalidPerUserDataMerge,
     AttendeeAllowedError, ResourceDeletedError, InvalidAttachmentOperation,
     ShareeAllowedError, DuplicatePrivateCommentsError, InvalidSplit,
-    AttachmentSizeTooLarge, UnknownTimezone)
+    AttachmentSizeTooLarge, UnknownTimezone, SetComponentOptions)
 from txdav.carddav.iaddressbookstore import (
     KindChangeNotAllowedError, GroupWithUnsharedAddressNotAllowedError
 )
@@ -2892,8 +2892,18 @@ def http_PUT(self, request):
                     "Can't parse calendar data: %s" % (str(e),)
                 ))
 
+            # Look for client fixes
+            ua = request.headers.getHeader("User-Agent")
+            client_fix_transp = matchClientFixes(config, ua)
+
+            # Setup options
+            options = {
+                SetComponentOptions.smartMerge: schedule_tag_match,
+                SetComponentOptions.clientFixTRANSP: client_fix_transp,
+            }
+
             try:
-                response = (yield self.storeComponent(component, smart_merge=schedule_tag_match))
+                response = (yield self.storeComponent(component, options=options))
             except ResourceDeletedError:
                 # This is OK - it just means the server deleted the resource during the PUT. We make it look
                 # like the PUT succeeded.

twistedcaldav/test/test_collectioncontents.py

@@ -48,7 +48,7 @@ def _getFakeMemcacheProtocol(self):
                    _getFakeMemcacheProtocol)
 
         # Need to not do implicit behavior during these tests
-        def _fakeDoImplicitScheduling(self, component, inserting, internal_state):
+        def _fakeDoImplicitScheduling(self, component, inserting, internal_state, options):
             return False, None, False, None
 
         self.patch(CalendarObject, "doImplicitScheduling",

twistedcaldav/test/test_util.py

@@ -16,8 +16,10 @@
 
 from txweb2.http_headers import Headers
 
+from twistedcaldav.config import ConfigDict
+from twistedcaldav.stdconfig import _updateClientFixes
+from twistedcaldav.util import bestAcceptType, userAgentProductTokens, matchClientFixes
 import twistedcaldav.test.util
-from twistedcaldav.util import bestAcceptType
 
 class AcceptType(twistedcaldav.test.util.TestCase):
     """
@@ -142,3 +144,78 @@ def test_bestAcceptType(self):
                 hdrs.addRawHeader(*hdr)
             check = bestAcceptType(hdrs.getHeader("accept"), allowedTypes)
             self.assertEqual(check, result, msg="Failed %s" % (title,))
+
+
+    def test_userAgentProductTokens(self):
+        """
+        Test that L{userAgentProductTokens} correctly parses a User-Agent header.
+        """
+        for hdr, result in (
+            # Valid syntax
+            ("Client/1.0", ["Client/1.0", ]),
+            ("Client/1.0 FooBar/2", ["Client/1.0", "FooBar/2", ]),
+            ("Client/1.0 (commentary here)", ["Client/1.0", ]),
+            ("Client/1.0 (FooBar/2)", ["Client/1.0", ]),
+            ("Client/1.0 (commentary here) FooBar/2", ["Client/1.0", "FooBar/2", ]),
+            ("Client/1.0 (commentary here) FooBar/2 (more commentary here) ", ["Client/1.0", "FooBar/2", ]),
+
+            # Invalid syntax
+            ("Client/1.0 (commentary here FooBar/2", ["Client/1.0", ]),
+            ("Client/1.0 commentary here) FooBar/2", ["Client/1.0", "commentary", "here)", "FooBar/2", ]),
+        ):
+            self.assertEqual(userAgentProductTokens(hdr), result, msg="Mismatch: {}".format(hdr))
+
+
+    def test_matchClientFixes(self):
+        """
+        Test that L{matchClientFixes} correctly identifies clients with matching fix tokens.
+        """
+        c = ConfigDict()
+        c.ClientFixes = {
+            "fix1": [
+                "Client/1\\.0.*",
+                "Client/1\\.1(\\..*)?",
+                "Client/2",
+            ],
+            "fix2": [
+                "Other/1\\.0.*",
+            ],
+        }
+        _updateClientFixes(c)
+        _updateClientFixes(c)
+
+        # Valid matches
+        for ua in (
+            "Client/1.0 FooBar/2",
+            "Client/1.0.1 FooBar/2",
+            "Client/1.0.1.1 FooBar/2",
+            "Client/1.1 FooBar/2",
+            "Client/1.1.1 FooBar/2",
+            "Client/2 FooBar/2",
+        ):
+            self.assertEqual(
+                matchClientFixes(c, ua),
+                set(("fix1",)),
+                msg="Did not match {}".format(ua),
+            )
+
+        # Valid non-matches
+        for ua in (
+            "Client/1 FooBar/2",
+            "Client/1.10 FooBar/2",
+            "Client/2.0 FooBar/2",
+            "Client/2.0.1 FooBar/2",
+            "Client FooBar/2",
+            "Client/3 FooBar/2",
+            "Client/3.0 FooBar/2",
+            "Client/10 FooBar/2",
+            "Client/10.0 FooBar/2",
+            "Client/10.0.1 FooBar/2",
+            "Client/10.0.1 (Client/1.0) FooBar/2",
+            "Client/10.0.1 (foo Client/1.0 bar) FooBar/2",
+        ):
+            self.assertEqual(
+                matchClientFixes(c, ua),
+                set(),
+                msg="Incorrectly matched {}".format(ua),
+            )

twistedcaldav/util.py

@@ -18,6 +18,7 @@
 import re
 import sys
 import base64
+import itertools
 
 from subprocess import Popen, PIPE, STDOUT
 from hashlib import md5, sha1
@@ -541,3 +542,57 @@ def bestAcceptType(accepts, allowedTypes):
                 result_qval = qval
 
     return result
+
+
+
+def userAgentProductTokens(user_agent):
+    """
+    Parse an HTTP User-Agent header to extract the product tokens and ignore
+    any parenthesized comment strings in the header.
+
+    @param user_agent: text of User-Agent header value
+    @type user_agent: L{str}
+
+    @return: list of product tokens extracted from the header
+    @rtype: L{list}
+    """
+
+    ua_hdr = user_agent.split()
+    ua_tokens = []
+    comment = False
+    for token in ua_hdr:
+        if comment:
+            if token.endswith(")"):
+                comment = False
+        elif token.startswith("("):
+            if not token.endswith(")"):
+                comment = True
+        else:
+            ua_tokens.append(token)
+
+    return ua_tokens
+
+
+
+def matchClientFixes(config, user_agent):
+    """
+    Given a user-agent string, see if it matches any of the configured client fixes.
+
+    @param config: the L{config} to match against.
+    @type config: L{ConfigDict}
+    @param user_agent: the HTTP User-Agent header value to test.
+    @type user_agent: L{str}
+    """
+
+    if len(config.ClientFixesCompiled) == 0 or not user_agent:
+        return set()
+
+    ua_tokens = userAgentProductTokens(user_agent)
+
+    client_fixes = set()
+    for fix, patterns in config.ClientFixesCompiled.items():
+        for pattern, token in itertools.product(patterns, ua_tokens):
+            if pattern.match(token) is not None:
+                client_fixes.add(fix)
+                break
+    return client_fixes

txdav/caldav/datastore/scheduling/icaldiff.py

@@ -27,30 +27,39 @@
 from txdav.caldav.datastore.scheduling.utils import normalizeCUAddr
 from txdav.caldav.datastore.scheduling.itip import iTipGenerator
 
-"""
-Class that handles diff'ing two calendar objects.
-"""
-
 __all__ = [
     "iCalDiff",
 ]
 
 log = Logger()
 
 class iCalDiff(object):
-
-    def __init__(self, oldcalendar, newcalendar, smart_merge):
+    """
+    This object is used for doing comparisons between two calendar objects to
+    work out what the changes are, in order to determine whether a scheduling
+    operation might be needed. The behavior will varying based on whether the
+    change is being triggered by an Organizer or an Attendee.
+    """
+
+    def __init__(self, oldcalendar, newcalendar, smart_merge, forceTRANSP=False):
         """
-
-        @param oldcalendar:
-        @type oldcalendar:
-        @param newcalendar:
-        @type newcalendar:
+        Note that this object will always duplicate the calendar objects when doing
+        comparisons so as not to change the calendar objects passed in.
+
+        @param oldcalendar: the existing calendar object to compare to
+        @type oldcalendar: L{Component}
+        @param newcalendar: the new calendar object
+        @type newcalendar: L{Component}
+        @param smart_merge: whether or not a "smart" CalDAV merge is done (If-Schedule-Tag-Match)
+        @type smart_merge: L{bool}
+        @param forceTRANSP: whether or not to apply a fix for clients failing to set TRANSP properly
+        @type forceTRANSP: L{bool}
         """
 
         self.oldcalendar = oldcalendar
         self.newcalendar = newcalendar
         self.smart_merge = smart_merge
+        self.forceTRANSP = forceTRANSP
 
 
     def organizerDiff(self):
@@ -501,6 +510,13 @@ def _transferAttendeeData(self, serverComponent, clientComponent, declines):
 
             replyNeeded = True
 
+            # Apply client fix only if the PARTSTAT was changed
+            if self.forceTRANSP:
+                if clientAttendee.parameterValue("PARTSTAT", "NEEDS-ACTION") in ("ACCEPTED", "TENTATIVE",):
+                    clientComponent.replaceProperty(Property("TRANSP", "OPAQUE"))
+                else:
+                    clientComponent.replaceProperty(Property("TRANSP", "TRANSPARENT"))
+
         if serverAttendee.parameterValue("RSVP", "FALSE") != clientAttendee.parameterValue("RSVP", "FALSE"):
             if clientAttendee.parameterValue("RSVP", "FALSE") == "FALSE":
                 try:

txdav/caldav/datastore/scheduling/implicit.py

@@ -37,6 +37,7 @@
 from txdav.caldav.datastore.scheduling.utils import getCalendarObjectForRecord
 from txdav.caldav.datastore.scheduling.work import ScheduleReplyWork, \
     ScheduleReplyCancelWork, ScheduleOrganizerWork, ScheduleOrganizerSendWork
+from txdav.caldav.icalendarstore import SetComponentOptions
 
 import collections
 
@@ -65,10 +66,11 @@ class ImplicitScheduler(object):
     STATUS_ORPHANED_CANCELLED_EVENT = 1
     STATUS_ORPHANED_EVENT = 2
 
-    def __init__(self, logItems=None):
+    def __init__(self, logItems=None, options=None):
 
         self.return_status = ImplicitScheduler.STATUS_OK
         self.logItems = logItems
+        self.options = options
         self.allowed_to_schedule = True
         self.suppress_refresh = False
 
@@ -1683,7 +1685,11 @@ def isAttendeeChangeInsignificant(self):
                     (caldav_namespace, "valid-attendee-change"),
                     "Cannot use an event when not listed as an attendee in the organizer's copy",
                 ))
-        differ = iCalDiff(oldcalendar, self.calendar, self.do_smart_merge)
+
+        # Check for a required client fix
+        forceTRANSP = SetComponentOptions.value(self.options, SetComponentOptions.clientFixTRANSP)
+
+        differ = iCalDiff(oldcalendar, self.calendar, self.do_smart_merge, forceTRANSP=forceTRANSP)
         return differ.attendeeMerge(self.attendee)