projectal.entity

   1"""
   2The base Entity class that all entities inherit from.
   3"""
   4
   5import copy
   6import logging
   7import sys
   8
   9import projectal
  10from projectal import api
  11
  12
  13class Entity(dict):
  14    """
  15    The parent class for all our entities, offering requests
  16    and validation for the fundamental create/read/update/delete
  17    operations.
  18
  19    This class (and all our entities) inherit from the builtin
  20    `dict` class. This means all entity classes can be used
  21    like standard Python dictionary objects, but we can also
  22    offer additional utility functions that operate on the
  23    instance itself (see `linkers` for an example). Any method
  24    that expects a `dict` can also consume an `Entity` subclass.
  25
  26    The class methods in this class can operate on one or more
  27    entities in one request. If the methods are called with
  28    lists (for batch operation), the output returned will also
  29    be a list. Otherwise, a single `Entity` subclass is returned.
  30
  31    Note for batch operations: a `ProjectalException` is raised
  32    if *any* of the entities fail during the operation. The
  33    changes will *still be saved to the database for the entities
  34    that did not fail*.
  35    """
  36
  37    #: Child classes must override these with their entity names
  38    _path = "entity"  # URL portion to api
  39    _name = "entity"
  40
  41    # And to which entities they link to
  42    _links = []
  43    _links_reverse = []
  44
  45    def __init__(self, data):
  46        dict.__init__(self, data)
  47        self._is_new = True
  48        self._link_def_by_key = {}
  49        self._link_def_by_name = {}
  50        self._create_link_defs()
  51        self._with_links = set()
  52
  53        self.__fetch = self.get
  54        self.get = self.__get
  55        self.update = self.__update
  56        self.delete = self.__delete
  57        self.history = self.__history
  58        self.__type_links()
  59        self.__old = copy.deepcopy(self)
  60
  61    # ----- LINKING -----
  62
  63    def _create_link_defs(self):
  64        for cls in self._links:
  65            self._add_link_def(cls)
  66        for cls in self._links_reverse:
  67            self._add_link_def(cls, reverse=True)
  68
  69    def _add_link_def(self, cls, reverse=False):
  70        """
  71        Each entity is accompanied by a dict with details about how to
  72        get access to the data of the link within the object. Subclasses
  73        can pass in customizations to this dict when their APIs differ.
  74
  75        reverse denotes a reverse linker, where extra work is done to
  76        reverse the relationship of the link internally so that it works.
  77        The backend only offers one side of the relationship.
  78        """
  79        d = {
  80            "name": cls._link_name,
  81            "link_key": cls._link_key or cls._link_name + "List",
  82            "data_name": cls._link_data_name,
  83            "type": cls._link_type,
  84            "entity": cls._link_entity or cls._link_name.capitalize(),
  85            "reverse": reverse,
  86        }
  87        self._link_def_by_key[d["link_key"]] = d
  88        self._link_def_by_name[d["name"]] = d
  89
  90    def _add_link(self, to_entity_name, to_link):
  91        self._link(to_entity_name, to_link, "add", batch_linking=False)
  92
  93    def _update_link(self, to_entity_name, to_link):
  94        self._link(to_entity_name, to_link, "update", batch_linking=False)
  95
  96    def _delete_link(self, to_entity_name, to_link):
  97        self._link(to_entity_name, to_link, "delete", batch_linking=False)
  98
  99    def _link(
 100        self, to_entity_name, to_link, operation, update_cache=True, batch_linking=True
 101    ):
 102        """
 103        `to_entity_name`: Destination entity name (e.g. 'staff')
 104
 105        `to_link`: List of Entities of the same type (and optional data) to link to
 106
 107        `operation`: `add`, `update`, `delete`
 108
 109        'update_cache': also modify the entity's internal representation of the links
 110        to match the operation that was done. Set this to False when replacing the
 111        list with a new one (i.e., when calling save() instead of a linker method).
 112
 113        'batch_linking': Enabled by default, batches any link
 114        updates required into composite API requests. If disabled
 115        a request will be executed for each link update.
 116        Recommended to leave enabled to increase performance.
 117        """
 118
 119        link_def = self._link_def_by_name[to_entity_name]
 120        to_key = link_def["link_key"]
 121
 122        if isinstance(to_link, dict) and link_def["type"] == list:
 123            # Convert input dict to list when link type is a list (we allow linking to single entity for convenience)
 124            to_link = [to_link]
 125
 126            # For cases where user passed in dict instead of Entity, we turn them into
 127            # Entity on their behalf.
 128            typed_list = []
 129            target_cls = getattr(sys.modules["projectal.entities"], link_def["entity"])
 130            for link in to_link:
 131                if not isinstance(link, target_cls):
 132                    typed_list.append(target_cls(link))
 133                else:
 134                    typed_list.append(link)
 135            to_link = typed_list
 136        else:
 137            # For everything else, we expect types to match.
 138            if not isinstance(to_link, link_def["type"]):
 139                raise api.UsageException(
 140                    "Expected link type to be {}. Got {}.".format(
 141                        link_def["type"], type(to_link)
 142                    )
 143                )
 144
 145        if not to_link:
 146            return
 147
 148        url = ""
 149        payload = {}
 150        request_list = []
 151        # Is it a reverse linker? If so, invert the relationship
 152        if link_def["reverse"]:
 153            for link in to_link:
 154                request_list.extend(
 155                    link._link(
 156                        self._name,
 157                        self,
 158                        operation,
 159                        update_cache,
 160                        batch_linking=batch_linking,
 161                    )
 162                )
 163        else:
 164            # Only keep UUID and the data attribute, if it has one
 165            def strip_payload(link):
 166                single = {"uuId": link["uuId"]}
 167                data_name = link_def.get("data_name")
 168                if data_name and data_name in link:
 169                    single[data_name] = copy.deepcopy(link[data_name])
 170                return single
 171
 172            # If batch linking is enabled and the entity to link is a list of entities,
 173            # a separate request must be constructed for each one because the final composite
 174            # request permits only one input per call
 175            url = "/api/{}/link/{}/{}".format(self._path, to_entity_name, operation)
 176            to_link_payload = None
 177            if isinstance(to_link, list):
 178                to_link_payload = []
 179                for link in to_link:
 180                    if batch_linking:
 181                        request_list.append(
 182                            {
 183                                "method": "POST",
 184                                "invoke": url,
 185                                "body": {
 186                                    "uuId": self["uuId"],
 187                                    to_key: [strip_payload(link)],
 188                                },
 189                            }
 190                        )
 191                    else:
 192                        to_link_payload.append(strip_payload(link))
 193            if isinstance(to_link, dict):
 194                if batch_linking:
 195                    request_list.append(
 196                        {
 197                            "method": "POST",
 198                            "invoke": url,
 199                            "body": {
 200                                "uuId": self["uuId"],
 201                                to_key: strip_payload(to_link),
 202                            },
 203                        }
 204                    )
 205                else:
 206                    to_link_payload = strip_payload(to_link)
 207
 208            if not batch_linking:
 209                payload = {"uuId": self["uuId"], to_key: to_link_payload}
 210                api.post(url, payload=payload)
 211
 212        if not update_cache:
 213            return request_list
 214
 215        # Set the initial state if first add. We need the type to be set to correctly update the cache
 216        if operation == "add" and self.get(to_key, None) is None:
 217            if link_def.get("type") == dict:
 218                self[to_key] = {}
 219            elif link_def.get("type") == list:
 220                self[to_key] = []
 221
 222        # Modify the entity object's cache of links to match the changes we pushed to the server.
 223        if isinstance(self.get(to_key, []), list):
 224            if operation == "add":
 225                # Sometimes the backend doesn't return a list when it has none. Create it.
 226                if to_key not in self:
 227                    self[to_key] = []
 228
 229                for to_entity in to_link:
 230                    self[to_key].append(to_entity)
 231            else:
 232                for to_entity in to_link:
 233                    # Find it in original list
 234                    for i, old in enumerate(self.get(to_key, [])):
 235                        if old["uuId"] == to_entity["uuId"]:
 236                            if operation == "update":
 237                                self[to_key][i] = to_entity
 238                            elif operation == "delete":
 239                                del self[to_key][i]
 240        if isinstance(self.get(to_key, None), dict):
 241            if operation in ["add", "update"]:
 242                self[to_key] = to_link
 243            elif operation == "delete":
 244                self[to_key] = None
 245
 246        # Update the "old" record of the link on the entity to avoid
 247        # flagging it for changes (link lists are not meant to be user editable).
 248        if to_key in self:
 249            self.__old[to_key] = self[to_key]
 250
 251        return request_list
 252
 253    # -----
 254
 255    @classmethod
 256    def create(
 257        cls,
 258        entities,
 259        params=None,
 260        batch_linking=True,
 261        disable_system_features=True,
 262        enable_system_features_on_exit=True,
 263    ):
 264        """
 265        Create one or more entities of the same type. The entity
 266        type is determined by the subclass calling this method.
 267
 268        `entities`: Can be a `dict` to create a single entity,
 269        or a list of `dict`s to create many entities in bulk.
 270
 271        `params`: Optional URL parameters that may apply to the
 272        entity's API (e.g: `?holder=1234`).
 273
 274        'batch_linking': Enabled by default, batches any link
 275        updates required into composite API requests. If disabled
 276        a request will be executed for each link update.
 277        Recommended to leave enabled to increase performance.
 278
 279        If input was a `dict`, returns an entity subclass. If input was
 280        a list of `dict`s, returns a list of entity subclasses.
 281
 282        ```
 283        # Example usage:
 284        projectal.Customer.create({'name': 'NewCustomer'})
 285        # returns Customer object
 286        ```
 287        """
 288
 289        if isinstance(entities, dict):
 290            # Dict input needs to be a list
 291            e_list = [entities]
 292        else:
 293            # We have a list of dicts already, the expected format
 294            e_list = entities
 295
 296        # Apply type
 297        typed_list = []
 298        for e in e_list:
 299            if not isinstance(e, Entity):
 300                # Start empty to correctly populate history
 301                new = cls({})
 302                new.update(e)
 303                typed_list.append(new)
 304            else:
 305                typed_list.append(e)
 306        e_list = typed_list
 307
 308        endpoint = "/api/{}/add".format(cls._path)
 309        if params:
 310            endpoint += params
 311        if not e_list:
 312            return []
 313
 314        # Strip links from payload
 315        payload = []
 316        keys = e_list[0]._link_def_by_key.keys()
 317        for e in e_list:
 318            cleancopy = copy.deepcopy(e)
 319            # Remove any fields that match a link key
 320            for key in keys:
 321                cleancopy.pop(key, None)
 322            payload.append(cleancopy)
 323
 324        objects = []
 325        for i in range(0, len(payload), projectal.chunk_size_write):
 326            chunk = payload[i : i + projectal.chunk_size_write]
 327            orig_chunk = e_list[i : i + projectal.chunk_size_write]
 328            response = api.post(endpoint, chunk)
 329            # Put uuId from response into each input dict
 330            for e, o, orig in zip(chunk, response, orig_chunk):
 331                orig["uuId"] = o["uuId"]
 332                orig.__old = copy.deepcopy(orig)
 333                # Delete links from the history in order to trigger a change on them after
 334                for key in orig._link_def_by_key:
 335                    orig.__old.pop(key, None)
 336                objects.append(orig)
 337
 338        # Detect and apply any link additions
 339        # if batch_linking is enabled, builds a list of link requests
 340        # needed for each entity, then executes them with composite
 341        # API requests
 342        link_request_batch = []
 343        for e in e_list:
 344            requests = e.__apply_link_changes(batch_linking=batch_linking)
 345            link_request_batch.extend(requests)
 346
 347        if len(link_request_batch) > 0 and batch_linking:
 348            for i in range(0, len(link_request_batch), 100):
 349                chunk = link_request_batch[i : i + 100]
 350                if disable_system_features:
 351                    chunk = [
 352                        {
 353                            "note": "Disable Scheduling",
 354                            "invoke": "PUT /api/system/features?entity=scheduling&action=DISABLE",
 355                        },
 356                        {
 357                            "note": "Disable Macros",
 358                            "invoke": "PUT /api/system/features?entity=macros&action=disable",
 359                        },
 360                    ] + chunk
 361                if not enable_system_features_on_exit:
 362                    chunk.append(
 363                        {
 364                            "note": "Exit script execution, and do not restore some disabled commands",
 365                            "invoke": "PUT /api/system/features?entity=script&action=EXIT",
 366                        }
 367                    )
 368                api.post("/api/composite", chunk)
 369
 370        if not isinstance(entities, list):
 371            return objects[0]
 372        return objects
 373
 374    @classmethod
 375    def _get_linkset(cls, links):
 376        """Get a set of link names we have been asked to fetch with. Raise an
 377        error if the requested link is not valid for this Entity type."""
 378        link_set = set()
 379        if links is not None:
 380            if isinstance(links, str) or not hasattr(links, "__iter__"):
 381                raise projectal.UsageException(
 382                    "Parameter 'links' must be a list or None."
 383                )
 384
 385            defs = cls({})._link_def_by_name
 386            for link in links:
 387                name = link.lower()
 388                if name not in defs:
 389                    raise projectal.UsageException(
 390                        "Link '{}' is invalid for {}".format(name, cls._name)
 391                    )
 392                link_set.add(name)
 393        return link_set
 394
 395    @classmethod
 396    def get(cls, entities, links=None, deleted_at=None):
 397        """
 398        Get one or more entities of the same type. The entity
 399        type is determined by the subclass calling this method.
 400
 401        `entities`: One of several formats containing the `uuId`s
 402        of the entities you want to get (see bottom for examples):
 403
 404        - `str` or list of `str`
 405        - `dict` or list of `dict` (with `uuId` key)
 406
 407        `links`: A case-insensitive list of entity names to fetch with
 408        this entity. For performance reasons, links are only returned
 409        on demand.
 410
 411        Links follow a common naming convention in the output with
 412        a *_List* suffix. E.g.:
 413        `links=['company', 'location']` will appear as `companyList` and
 414        `locationList` in the response.
 415        ```
 416        # Example usage:
 417        # str
 418        projectal.Project.get('1b21e445-f29a-4a9f-95ff-fe253a3e1b11')
 419
 420        # list of str
 421        ids = ['1b21e445-f29a...', '1b21e445-f29a...', '1b21e445-f29a...']
 422        projectal.Project.get(ids)
 423
 424        # dict
 425        project = project.Project.create({'name': 'MyProject'})
 426        # project = {'uuId': '1b21e445-f29a...', 'name': 'MyProject', ...}
 427        projectal.Project.get(project)
 428
 429        # list of dicts (e.g. from a query)
 430        # projects = [{'uuId': '1b21e445-f29a...'}, {'uuId': '1b21e445-f29a...'}, ...]
 431        project.Project.get(projects)
 432
 433        # str with links
 434        projectal.Project.get('1b21e445-f29a...', 'links=['company', 'location']')
 435        ```
 436
 437        `deleted_at`: Include this parameter to get a deleted entity.
 438        This value should be a UTC timestamp from a webhook delete event.
 439        """
 440        link_set = cls._get_linkset(links)
 441
 442        if isinstance(entities, str):
 443            # String input is a uuId
 444            payload = [{"uuId": entities}]
 445        elif isinstance(entities, dict):
 446            # Dict input needs to be a list
 447            payload = [entities]
 448        elif isinstance(entities, list):
 449            # List input can be a list of uuIds or list of dicts
 450            # If uuIds (strings), convert to list of dicts
 451            if len(entities) > 0 and isinstance(entities[0], str):
 452                payload = [{"uuId": uuId} for uuId in entities]
 453            else:
 454                # Already expected format
 455                payload = entities
 456        else:
 457            # We have a list of dicts already, the expected format
 458            payload = entities
 459
 460        if deleted_at:
 461            if not isinstance(deleted_at, int):
 462                raise projectal.UsageException("deleted_at must be a number")
 463
 464        url = "/api/{}/get".format(cls._path)
 465        params = []
 466        params.append("links={}".format(",".join(links))) if links else None
 467        params.append("epoch={}".format(deleted_at - 1)) if deleted_at else None
 468        if len(params) > 0:
 469            url += "?" + "&".join(params)
 470
 471        # We only need to send over the uuIds
 472        payload = [{"uuId": e["uuId"]} for e in payload]
 473        if not payload:
 474            return []
 475        objects = []
 476        for i in range(0, len(payload), projectal.chunk_size_read):
 477            chunk = payload[i : i + projectal.chunk_size_read]
 478            dicts = api.post(url, chunk)
 479            for d in dicts:
 480                obj = cls(d)
 481                obj._with_links.update(link_set)
 482                obj._is_new = False
 483                # Create default fields for links we ask for. Workaround for backend
 484                # sometimes omitting links if no links exist.
 485                for link_name in link_set:
 486                    link_def = obj._link_def_by_name[link_name]
 487                    if link_def["link_key"] not in obj:
 488                        if link_def["type"] == dict:
 489                            obj.set_readonly(link_def["link_key"], None)
 490                        else:
 491                            obj.set_readonly(link_def["link_key"], link_def["type"]())
 492                objects.append(obj)
 493
 494        if not isinstance(entities, list):
 495            return objects[0]
 496        return objects
 497
 498    def __get(self, *args, **kwargs):
 499        """Use the dict get for instances."""
 500        return super(Entity, self).get(*args, **kwargs)
 501
 502    @classmethod
 503    def update(
 504        cls,
 505        entities,
 506        batch_linking=True,
 507        disable_system_features=True,
 508        enable_system_features_on_exit=True,
 509    ):
 510        """
 511        Save one or more entities of the same type. The entity
 512        type is determined by the subclass calling this method.
 513        Only the fields that have been modifier will be sent
 514        to the server as part of the request.
 515
 516        `entities`: Can be a `dict` to update a single entity,
 517        or a list of `dict`s to update many entities in bulk.
 518
 519        'batch_linking': Enabled by default, batches any link
 520        updates required into composite API requests. If disabled
 521        a request will be executed for each link update.
 522        Recommended to leave enabled to increase performance.
 523
 524        Returns `True` if all entities update successfully.
 525
 526        ```
 527        # Example usage:
 528        rebate = projectal.Rebate.create({'name': 'Rebate2022', 'rebate': 0.2})
 529        rebate['name'] = 'Rebate2024'
 530        projectal.Rebate.update(rebate)
 531        # Returns True. New rebate name has been saved.
 532        ```
 533        """
 534        if isinstance(entities, dict):
 535            e_list = [entities]
 536        else:
 537            e_list = entities
 538
 539        # allows for filtering of link keys
 540        typed_list = []
 541        for e in e_list:
 542            if not isinstance(e, Entity):
 543                new = cls({})
 544                new.update(e)
 545                typed_list.append(new)
 546            else:
 547                typed_list.append(e)
 548        e_list = typed_list
 549
 550        # Reduce the list to only modified entities and their modified fields.
 551        # Only do this to an Entity subclass - the consumer may have passed
 552        # in a dict of changes on their own.
 553        payload = []
 554
 555        for e in e_list:
 556            if isinstance(e, Entity):
 557                changes = e._changes_internal()
 558                if changes:
 559                    changes["uuId"] = e["uuId"]
 560                    payload.append(changes)
 561            else:
 562                payload.append(e)
 563        if payload:
 564            for i in range(0, len(payload), projectal.chunk_size_write):
 565                chunk = payload[i : i + projectal.chunk_size_write]
 566                api.put("/api/{}/update".format(cls._path), chunk)
 567
 568        # Detect and apply any link changes
 569        # if batch_linking is enabled, builds a list of link requests
 570        # from the changes of each entity, then executes
 571        # composite API requests with those changes
 572        link_request_batch = []
 573        for e in e_list:
 574            if isinstance(e, Entity):
 575                requests = e.__apply_link_changes(batch_linking=batch_linking)
 576                link_request_batch.extend(requests)
 577
 578        if len(link_request_batch) > 0 and batch_linking:
 579            for i in range(0, len(link_request_batch), 100):
 580                chunk = link_request_batch[i : i + 100]
 581                if disable_system_features:
 582                    chunk = [
 583                        {
 584                            "note": "Disable Scheduling",
 585                            "invoke": "PUT /api/system/features?entity=scheduling&action=DISABLE",
 586                        },
 587                        {
 588                            "note": "Disable Macros",
 589                            "invoke": "PUT /api/system/features?entity=macros&action=disable",
 590                        },
 591                    ] + chunk
 592                if not enable_system_features_on_exit:
 593                    chunk.append(
 594                        {
 595                            "note": "Exit script execution, and do not restore some disabled commands",
 596                            "invoke": "PUT /api/system/features?entity=script&action=EXIT",
 597                        }
 598                    )
 599                api.post("/api/composite", chunk)
 600
 601        return True
 602
 603    def __update(self, *args, **kwargs):
 604        """Use the dict update for instances."""
 605        return super(Entity, self).update(*args, **kwargs)
 606
 607    def save(
 608        self,
 609        batch_linking=True,
 610        disable_system_features=True,
 611        enable_system_features_on_exit=True,
 612    ):
 613        """Calls `update()` on this instance of the entity, saving
 614        it to the database."""
 615        return self.__class__.update(
 616            self, batch_linking, disable_system_features, enable_system_features_on_exit
 617        )
 618
 619    @classmethod
 620    def delete(cls, entities):
 621        """
 622        Delete one or more entities of the same type. The entity
 623        type is determined by the subclass calling this method.
 624
 625        `entities`: See `Entity.get()` for expected formats.
 626
 627        ```
 628        # Example usage:
 629        ids = ['1b21e445-f29a...', '1b21e445-f29a...', '1b21e445-f29a...']
 630        projectal.Customer.delete(ids)
 631        ```
 632        """
 633        if isinstance(entities, str):
 634            # String input is a uuId
 635            payload = [{"uuId": entities}]
 636        elif isinstance(entities, dict):
 637            # Dict input needs to be a list
 638            payload = [entities]
 639        elif isinstance(entities, list):
 640            # List input can be a list of uuIds or list of dicts
 641            # If uuIds (strings), convert to list of dicts
 642            if len(entities) > 0 and isinstance(entities[0], str):
 643                payload = [{"uuId": uuId} for uuId in entities]
 644            else:
 645                # Already expected format
 646                payload = entities
 647        else:
 648            # We have a list of dicts already, the expected format
 649            payload = entities
 650
 651        # We only need to send over the uuIds
 652        payload = [{"uuId": e["uuId"]} for e in payload]
 653        if not payload:
 654            return True
 655        for i in range(0, len(payload), projectal.chunk_size_write):
 656            chunk = payload[i : i + projectal.chunk_size_write]
 657            api.delete("/api/{}/delete".format(cls._path), chunk)
 658        return True
 659
 660    def __delete(self):
 661        """Let an instance delete itself."""
 662        return self.__class__.delete(self)
 663
 664    def clone(self, entity):
 665        """
 666        Clones an entity and returns its `uuId`.
 667
 668        Each entity has its own set of required values when cloning.
 669        Check the API documentation of that entity for details.
 670        """
 671        url = "/api/{}/clone?reference={}".format(self._path, self["uuId"])
 672        response = api.post(url, entity)
 673        return response["jobClue"]["uuId"]
 674
 675    @classmethod
 676    def history(cls, UUID, start=0, limit=-1, order="desc", epoch=None, event=None):
 677        """
 678        Returns an ordered list of all changes made to the entity.
 679
 680        `UUID`: the UUID of the entity.
 681
 682        `start`: Start index for pagination (default: `0`).
 683
 684        `limit`: Number of results to include for pagination. Use
 685        `-1` to return the entire history (default: `-1`).
 686
 687        `order`: `asc` or `desc` (default: `desc` (index 0 is newest))
 688
 689        `epoch`: only return the history UP TO epoch date
 690
 691        `event`:
 692        """
 693        url = "/api/{}/history?holder={}&".format(cls._path, UUID)
 694        params = []
 695        params.append("start={}".format(start))
 696        params.append("limit={}".format(limit))
 697        params.append("order={}".format(order))
 698        params.append("epoch={}".format(epoch)) if epoch else None
 699        params.append("event={}".format(event)) if event else None
 700        url += "&".join(params)
 701        return api.get(url)
 702
 703    def __history(self, **kwargs):
 704        """Get history of instance."""
 705        return self.__class__.history(self["uuId"], **kwargs)
 706
 707    @classmethod
 708    def list(cls, expand=False, links=None):
 709        """Return a list of all entity UUIDs of this type.
 710
 711        You may pass in `expand=True` to get full Entity objects
 712        instead, but be aware this may be very slow if you have
 713        thousands of objects.
 714
 715        If you are expanding the objects, you may further expand
 716        the results with `links`.
 717        """
 718
 719        payload = {
 720            "name": "List all entities of type {}".format(cls._name.upper()),
 721            "type": "msql",
 722            "start": 0,
 723            "limit": -1,
 724            "select": [["{}.uuId".format(cls._name.upper())]],
 725        }
 726        ids = api.query(payload)
 727        ids = [id[0] for id in ids]
 728        if ids:
 729            return cls.get(ids, links=links) if expand else ids
 730        return []
 731
 732    @classmethod
 733    def match(cls, field, term, links=None):
 734        """Find entities where `field`=`term` (exact match), optionally
 735        expanding the results with `links`.
 736
 737        Relies on `Entity.query()` with a pre-built set of rules.
 738        ```
 739        projects = projectal.Project.match('identifier', 'zmb-005')
 740        ```
 741        """
 742        filter = [["{}.{}".format(cls._name.upper(), field), "eq", term]]
 743        return cls.query(filter, links)
 744
 745    @classmethod
 746    def match_startswith(cls, field, term, links=None):
 747        """Find entities where `field` starts with the text `term`,
 748        optionally expanding the results with `links`.
 749
 750        Relies on `Entity.query()` with a pre-built set of rules.
 751        ```
 752        projects = projectal.Project.match_startswith('name', 'Zomb')
 753        ```
 754        """
 755        filter = [["{}.{}".format(cls._name.upper(), field), "prefix", term]]
 756        return cls.query(filter, links)
 757
 758    @classmethod
 759    def match_endswith(cls, field, term, links=None):
 760        """Find entities where `field` ends with the text `term`,
 761        optionally expanding the results with `links`.
 762
 763        Relies on `Entity.query()` with a pre-built set of rules.
 764        ```
 765        projects = projectal.Project.match_endswith('identifier', '-2023')
 766        ```
 767        """
 768        term = "(?i).*{}$".format(term)
 769        filter = [["{}.{}".format(cls._name.upper(), field), "regex", term]]
 770        return cls.query(filter, links)
 771
 772    @classmethod
 773    def match_one(cls, field, term, links=None):
 774        """Convenience function for match(). Returns the first match or None."""
 775        matches = cls.match(field, term, links)
 776        if matches:
 777            return matches[0]
 778
 779    @classmethod
 780    def match_startswith_one(cls, field, term, links=None):
 781        """Convenience function for match_startswith(). Returns the first match or None."""
 782        matches = cls.match_startswith(field, term, links)
 783        if matches:
 784            return matches[0]
 785
 786    @classmethod
 787    def match_endswith_one(cls, field, term, links=None):
 788        """Convenience function for match_endswith(). Returns the first match or None."""
 789        matches = cls.match_endswith(field, term, links)
 790        if matches:
 791            return matches[0]
 792
 793    @classmethod
 794    def search(cls, fields=None, term="", case_sensitive=True, links=None):
 795        """Find entities that contain the text `term` within `fields`.
 796        `fields` is a list of field names to target in the search.
 797
 798        `case_sensitive`: Optionally turn off case sensitivity in the search.
 799
 800        Relies on `Entity.query()` with a pre-built set of rules.
 801        ```
 802        projects = projectal.Project.search(['name', 'description'], 'zombie')
 803        ```
 804        """
 805        filter = []
 806        term = "(?{}).*{}.*".format("" if case_sensitive else "?", term)
 807        for field in fields:
 808            filter.append(["{}.{}".format(cls._name.upper(), field), "regex", term])
 809        filter = ["_or_", filter]
 810        return cls.query(filter, links)
 811
 812    @classmethod
 813    def query(cls, filter, links=None, timeout=30):
 814        """Run a query on this entity with the supplied filter.
 815
 816        The query is already set up to target this entity type, and the
 817        results will be converted into full objects when found, optionally
 818        expanded with the `links` provided. You only need to supply a
 819        filter to reduce the result set.
 820
 821        See [the filter documentation](https://projectal.com/docs/v1.1.1#section/Filter-section)
 822        for a detailed overview of the kinds of filters you can construct.
 823        """
 824        ids = []
 825        request_completed = False
 826        limit = projectal.query_chunk_size
 827        start = 0
 828        while not request_completed:
 829            payload = {
 830                "name": "Python library entity query ({})".format(cls._name.upper()),
 831                "type": "msql",
 832                "start": start,
 833                "limit": limit,
 834                "select": [["{}.uuId".format(cls._name.upper())]],
 835                "filter": filter,
 836                "timeout": timeout,
 837            }
 838            result = projectal.query(payload)
 839            ids.extend(result)
 840            if len(result) < limit:
 841                request_completed = True
 842            else:
 843                start += limit
 844
 845        ids = [id[0] for id in ids]
 846        if ids:
 847            return cls.get(ids, links=links)
 848        return []
 849
 850    def profile_get(self, key):
 851        """Get the profile (metadata) stored for this entity at `key`."""
 852        return projectal.profile.get(key, self.__class__._name.lower(), self["uuId"])
 853
 854    def profile_set(self, key, data):
 855        """Set the profile (metadata) stored for this entity at `key`. The contents
 856        of `data` will completely overwrite the existing data dictionary."""
 857        return projectal.profile.set(
 858            key, self.__class__._name.lower(), self["uuId"], data
 859        )
 860
 861    def __type_links(self):
 862        """Find links and turn their dicts into typed objects matching their Entity type."""
 863
 864        for key, _def in self._link_def_by_key.items():
 865            if key in self:
 866                cls = getattr(projectal, _def["entity"])
 867                if _def["type"] == list:
 868                    as_obj = []
 869                    for link in self[key]:
 870                        as_obj.append(cls(link))
 871                elif _def["type"] == dict:
 872                    as_obj = cls(self[key])
 873                else:
 874                    raise projectal.UsageException("Unexpected link type")
 875                self[key] = as_obj
 876
 877    def changes(self):
 878        """Return a dict containing the fields that have changed since fetching the object.
 879        Dict values contain both the old and new values. E.g.: {'old': 'original', 'new': 'current'}.
 880
 881        In the case of link lists, there are three values: added, removed, updated. Only links with
 882        a data attribute can end up in the updated list, and the old/new dictionary is placed within
 883        that data attribute. E.g. for a staff-resource link:
 884        'updated': [{
 885            'uuId': '24eb4c31-0f92-49d1-8b4d-507ab939003e',
 886            'resourceLink': {'quantity': {'old': 2, 'new': 5}}
 887        }]
 888        """
 889        changed = {}
 890        for key in self.keys():
 891            link_def = self._link_def_by_key.get(key)
 892            if link_def:
 893                changes = self._changes_for_link_list(link_def, key)
 894                # Only add it if something in it changed
 895                for action in changes.values():
 896                    if len(action):
 897                        changed[key] = changes
 898                        break
 899            elif key not in self.__old and self[key] is not None:
 900                changed[key] = {"old": None, "new": self[key]}
 901            elif self.__old.get(key) != self[key]:
 902                changed[key] = {"old": self.__old.get(key), "new": self[key]}
 903        return changed
 904
 905    def _changes_for_link_list(self, link_def, key):
 906        changes = self.__apply_list(link_def, report_only=True)
 907        data_key = link_def["data_name"]
 908
 909        # For linked entities, we will only report their UUID, name (if it has one),
 910        # and the content of their data attribute (if it has one).
 911        def get_slim_list(entities):
 912            slim = []
 913            if isinstance(entities, dict):
 914                entities = [entities]
 915            for e in entities:
 916                fields = {"uuId": e["uuId"]}
 917                name = e.get("name")
 918                if name:
 919                    fields["name"] = e["name"]
 920                if data_key and e[data_key]:
 921                    fields[data_key] = e[data_key]
 922                slim.append(fields)
 923            return slim
 924
 925        out = {
 926            "added": get_slim_list(changes.get("add", [])),
 927            "updated": [],
 928            "removed": get_slim_list(changes.get("remove", [])),
 929        }
 930
 931        updated = changes.get("update", [])
 932        if updated:
 933            before_map = {}
 934            for entity in self.__old.get(key):
 935                before_map[entity["uuId"]] = entity
 936
 937            for entity in updated:
 938                old_data = before_map[entity["uuId"]][data_key]
 939                new_data = entity[data_key]
 940                diff = {}
 941                for key in new_data.keys():
 942                    if key not in old_data and new_data[key] is not None:
 943                        diff[key] = {"old": None, "new": new_data[key]}
 944                    elif old_data.get(key) != new_data[key]:
 945                        diff[key] = {"old": old_data.get(key), "new": new_data[key]}
 946                out["updated"].append({"uuId": entity["uuId"], data_key: diff})
 947        return out
 948
 949    def _changes_internal(self):
 950        """Return a dict containing only the fields that have changed and their current value,
 951        without any link data.
 952
 953        This method is used internally to strip payloads down to only the fields that have changed.
 954        """
 955        changed = {}
 956        for key in self.keys():
 957            # We don't deal with link or link data changes here. We only want standard fields.
 958            if key in self._link_def_by_key:
 959                continue
 960            if key not in self.__old and self[key] is not None:
 961                changed[key] = self[key]
 962            elif self.__old.get(key) != self[key]:
 963                changed[key] = self[key]
 964        return changed
 965
 966    def set_readonly(self, key, value):
 967        """Set a field on this Entity that will not be sent over to the
 968        server on update unless modified."""
 969        self[key] = value
 970        self.__old[key] = value
 971
 972    # --- Link management ---
 973
 974    @staticmethod
 975    def __link_data_differs(have_link, want_link, data_key):
 976        if data_key:
 977            if "uuId" in have_link[data_key]:
 978                del have_link[data_key]["uuId"]
 979            if "uuId" in want_link[data_key]:
 980                del want_link[data_key]["uuId"]
 981            return have_link[data_key] != want_link[data_key]
 982
 983        # Links without data never differ
 984        return False
 985
 986    def __apply_link_changes(self, batch_linking=True):
 987        """Send each link list to the conflict resolver. If we detect
 988        that the entity was not fetched with that link, we do the fetch
 989        first and use the result as the basis for comparison."""
 990
 991        # Find which lists belong to links but were not fetched so we can fetch them
 992        need = []
 993        find_list = []
 994        if not self._is_new:
 995            for link in self._link_def_by_key.values():
 996                if link["link_key"] in self and link["name"] not in self._with_links:
 997                    need.append(link["name"])
 998                    find_list.append(link["link_key"])
 999
1000        if len(need):
1001            logging.warning(
1002                "Entity links were modified but entity not fetched with links. "
1003                "For better performance, include the links when getting the entity."
1004            )
1005            logging.warning(
1006                "Fetching {} again with missing links: {}".format(
1007                    self._name.upper(), ",".join(need)
1008                )
1009            )
1010            new = self.__fetch(self, links=need)
1011            for _list in find_list:
1012                self.__old[_list] = copy.deepcopy(new.get(_list, []))
1013
1014        # if batch_linking is enabled, builds a list of link requests
1015        # for each link definition of the calling entity then returns the list
1016        request_list = []
1017        for link_def in self._link_def_by_key.values():
1018            link_def_requests = self.__apply_list(link_def, batch_linking=batch_linking)
1019            if batch_linking:
1020                request_list.extend(link_def_requests)
1021        return request_list
1022
1023    def __apply_list(self, link_def, report_only=False, batch_linking=True):
1024        """Automatically resolve differences and issue the correct sequence of
1025        link/unlink/relink for the link list to result in the supplied list
1026        of entities.
1027
1028        report_only will not make any changes to the data or issue network requests.
1029        Instead, it returns the three lists of changes (add, update, delete).
1030        """
1031        to_add = []
1032        to_remove = []
1033        to_update = []
1034        should_only_have = set()
1035        link_key = link_def["link_key"]
1036
1037        if link_def["type"] == list:
1038            want_entities = self.get(link_key, [])
1039            have_entities = self.__old.get(link_key, [])
1040
1041            if not isinstance(want_entities, list):
1042                raise api.UsageException(
1043                    "Expecting '{}' to be {}. Found {} instead.".format(
1044                        link_key,
1045                        link_def["type"].__name__,
1046                        type(want_entities).__name__,
1047                    )
1048                )
1049
1050            for want_entity in want_entities:
1051                if want_entity["uuId"] in should_only_have:
1052                    raise api.UsageException(
1053                        "Duplicate {} in {}".format(link_def["name"], link_key)
1054                    )
1055                should_only_have.add(want_entity["uuId"])
1056                have = False
1057                for have_entity in have_entities:
1058                    if have_entity["uuId"] == want_entity["uuId"]:
1059                        have = True
1060                        data_name = link_def.get("data_name")
1061                        if data_name and self.__link_data_differs(
1062                            have_entity, want_entity, data_name
1063                        ):
1064                            to_update.append(want_entity)
1065                if not have:
1066                    to_add.append(want_entity)
1067            for have_entity in have_entities:
1068                if have_entity["uuId"] not in should_only_have:
1069                    to_remove.append(have_entity)
1070        elif link_def["type"] == dict:
1071            # Note: dict type does not implement updates as we have no dict links
1072            # that support update (yet?).
1073            want_entity = self.get(link_key, None)
1074            have_entity = self.__old.get(link_key, None)
1075
1076            if want_entity is not None and not isinstance(want_entity, dict):
1077                raise api.UsageException(
1078                    "Expecting '{}' to be {}. Found {} instead.".format(
1079                        link_key, link_def["type"].__name__, type(have_entity).__name__
1080                    )
1081                )
1082
1083            if want_entity:
1084                if have_entity:
1085                    if want_entity["uuId"] != have_entity["uuId"]:
1086                        to_remove = have_entity
1087                        to_add = want_entity
1088                else:
1089                    to_add = want_entity
1090            if not want_entity:
1091                if have_entity:
1092                    to_remove = have_entity
1093
1094            want_entities = want_entity
1095        else:
1096            # Would be an error in this library if we reach here
1097            raise projectal.UnsupportedException("This type does not support linking")
1098
1099        # if batch_linking is enabled, builds a list of requests
1100        # from each link method
1101        if not report_only:
1102            request_list = []
1103            if to_remove:
1104                delete_requests = self._link(
1105                    link_def["name"],
1106                    to_remove,
1107                    "delete",
1108                    update_cache=False,
1109                    batch_linking=batch_linking,
1110                )
1111                request_list.extend(delete_requests)
1112            if to_update:
1113                update_requests = self._link(
1114                    link_def["name"],
1115                    to_update,
1116                    "update",
1117                    update_cache=False,
1118                    batch_linking=batch_linking,
1119                )
1120                request_list.extend(update_requests)
1121            if to_add:
1122                add_requests = self._link(
1123                    link_def["name"],
1124                    to_add,
1125                    "add",
1126                    update_cache=False,
1127                    batch_linking=batch_linking,
1128                )
1129                request_list.extend(add_requests)
1130            self.__old[link_key] = copy.deepcopy(want_entities)
1131            return request_list
1132        else:
1133            changes = {}
1134            if to_remove:
1135                changes["remove"] = to_remove
1136            if to_update:
1137                changes["update"] = to_update
1138            if to_add:
1139                changes["add"] = to_add
1140            return changes
1141
1142    @classmethod
1143    def get_link_definitions(cls):
1144        return cls({})._link_def_by_name
1145
1146    # --- ---
1147
1148    def entity_name(self):
1149        return self._name.capitalize()