Merge pull request #4 from usnistgov/fix/dbio-missing-api

DAP Publishing: fix DAP API issues

python/nistoar/midas/dap/service/mds3.py

@@ -17,7 +17,7 @@
 Support for the web service frontend is provided via :py:class:`DAPApp` class, an implementation
 of the WSGI-based :ref:class:`~nistoar.pdr.publish.service.wsgi.SubApp`.
 """
-import os, re, pkg_resources
+import os, re, pkg_resources, random, string
 from logging import Logger
 from collections import OrderedDict
 from collections.abc import Mapping, MutableMapping, Sequence, Callable
@@ -92,6 +92,10 @@
 RES_DELIM = const.RESONLY_EXTENSION.lstrip('/')
 EXTSCHPROP = "_extensionSchemas"
 
+def random_id(prefix: str="", n: int=8):
+ r = ''.join(random.choices(string.ascii_uppercase + string.digits, k=n))
+ return prefix+r
+
 class DAPService(ProjectService):
  """
  a project record request broker class for DAP records. 
@@ -481,13 +485,15 @@ def update_data(self, id, newdata, part=None, message="", _prec=None):
  """
  return self._update_data(id, newdata, part, replace=False, message="", prec=_prec)
 
- def clear_data(self, id, part=None, _prec=None):
+ def clear_data(self, id, part=None, message: str=None, _prec=None) -> bool:
  """
  remove the stored data content of the record and reset it to its defaults. 
  :param str id: the identifier for the record whose data should be cleared.
  :param stt part: the slash-delimited pointer to an internal data property. If provided, 
  only that property will be cleared (either removed or set to an initial
  default).
+ :return: True the data was properly cleared; return False if ``part`` was specified but does not
+ yet exist in the data.
  :param ProjectRecord prec: the previously fetched and possibly updated record corresponding to 
  ``id``. If this is not provided, the record will by fetched anew based on 
  the ``id``. 
@@ -509,44 +515,61 @@ def clear_data(self, id, part=None, _prec=None):
  self._store.load_from(nerd)
  nerd = self._store.open(id)
 
+ provact = None
  try:
  if part:
  what = part
  if part == "authors":
+ if nerd.authors.count == 0:
+ return False
  nerd.authors.empty()
  elif part == "references":
+ if nerd.references.count == 0:
+ return False
  nerd.references.empty()
  elif part == FILE_DELIM:
+ if nerd.files.count == 0:
+ return False
  what = "files"
  nerd.files.empty()
  elif part == LINK_DELIM:
+ if nerd.nonfiles.count == 0:
+ return False
  what = "links"
  nerd.nonfiles.empty()
  elif part == "components":
+ if nerd.nonfiles.count == 0 and nerd.files.count == 0:
+ return False
  nerd.files.empty()
  nerd.nonfiles.empty()
- elif part in "title rights disclaimer description".split():
+ elif part in "title rights disclaimer description landingPage keyword".split():
  resmd = nerd.get_res_data()
+ if part not in resmd:
+ return False
  del resmd[part]
  nerd.replace_res_data(resmd)
  else:
- raise PartNotAccessible(_prec.id, path, "Clearing %s not allowed" % path)
+ raise PartNotAccessible(_prec.id, part, "Clearing %s not allowed" % part)
 
- provact = Action(Action.PATCH, _prec.id, self.who, "clearing "+what)
+ if not message:
+ message = "clearing "+what
+ provact = Action(Action.PATCH, _prec.id, self.who, message)
  part = ("/"+part) if part.startswith("pdr:") else ("."+part)
  provact.add_subaction(Action(Action.DELETE, _prec.id+"#data"+part, self.who,
- "clearing "+what))
- prec.status.act(self.STATUS_ACTION_CLEAR, "cleared "+what)
+ message))
+ _prec.status.act(self.STATUS_ACTION_CLEAR, "cleared "+what)
 
  else:
  nerd.authors.empty()
  nerd.references.empty()
  nerd.files.empty()
  nerd.nonfiles.empty()
- nerd.replace_res_data(self._new_data_for(_prec.id, prec.meta))
+ nerd.replace_res_data(self._new_data_for(_prec.id, _prec.meta))
 
- provact = Action(Action.PATCH, _prec.id, self.who, "clearing all NERDm data")
- prec.status.act(self.STATUS_ACTION_CLEAR, "cleared all NERDm data")
+ if not message:
+ message = "clearing all NERDm data"
+ provact = Action(Action.PATCH, _prec.id, self.who, message)
+ _prec.status.act(self.STATUS_ACTION_CLEAR, "cleared all NERDm data")
 
  except PartNotAccessible:
  # client request error; don't record action
@@ -555,28 +578,30 @@ def clear_data(self, id, part=None, _prec=None):
  except Exception as ex:
  self.log.error("Failed to clear requested NERDm data, %s: %s", _prec.id, str(ex))
  self.log.warning("Partial update is possible")
- provact.message = "Failed to clear requested NERDm data"
- self._record_action(provact)
+ if provact:
+ provact.message = "Failed to clear requested NERDm data"
+ self._record_action(provact)
 
- prec.status.act(self.STATUS_ACTION_CLEAR, "Failed to clear NERDm data")
- prec.set_state(status.EDIT)
- prec.data = self._summarize(nerd)
- self._try_save(prec)
+ _prec.status.act(self.STATUS_ACTION_CLEAR, "Failed to clear NERDm data")
+ _prec.set_state(status.EDIT)
+ _prec.data = self._summarize(nerd)
+ self._try_save(_prec)
  raise
 
- prec.data = self._summarize(nerd)
+ _prec.data = self._summarize(nerd)
  if set_state:
- prec.status.set_state(status.EDIT)
+ _prec.status.set_state(status.EDIT)
 
  try:
- prec.save()
+ _prec.save()
 
  except Exception as ex:
  self.log.error("Failed to saved DBIO record, %s: %s", prec.id, str(ex))
  raise
 
  finally:
- self._record_action(provact) 
+ self._record_action(provact)
+ return True
 
 
  def _update_data(self, id, newdata, part=None, prec=None, nerd=None, replace=False, message=""):
@@ -777,6 +802,11 @@ def put_each_into(data, objlist):
  for fmd in files:
  nerd.files.set_file_at(fmd)
 
+ except InvalidUpdate as ex:
+ self.log.error("Invalid update to NERDm data not saved: %s: %s", prec.id, str(ex))
+ if ex.errors:
+ self.log.error("Errors include:\n "+("\n ".join([str(e) for e in ex.errors])))
+ raise
  except Exception as ex:
  provact.message = "Failed to save NERDm data update due to internal error"
  self.log.error("Failed to save NERDm metadata: "+str(ex))
@@ -1017,6 +1047,11 @@ def _update_part_nerd(self, path: str, prec: ProjectRecord, nerd: NERDResource,
  except PartNotAccessible:
  # client request error; don't record action
  raise
+ except InvalidUpdate as ex:
+ self.log.error("Invalid update to NERDm data not saved: %s: %s", prec.id, str(ex))
+ if ex.errors:
+ self.log.error("Errors include:\n "+("\n ".join([str(e) for e in ex.errors])))
+ raise
  except Exception as ex:
  self.log.error("Failed to save update to NERDm data, %s: %s", prec.id, str(ex))
  self.log.warning("Partial update is possible")
@@ -1697,7 +1732,8 @@ def _moderate_author(self, auth, doval=True):
  return auth
 
  _refprops = set(("@id _schema _extensionSchemas title abbrev proxyFor location label "+
- "description citation refType doi inPreparation vol volNumber pages publishYear").split())
+ "description citation refType doi inPreparation vol volNumber pages "+
+ "authors publishYear").split())
  _reftypes = set(("IsDocumentedBy IsSupplementTo IsSupplementedBy IsCitedBy Cites IsReviewedBy "+
  "IsReferencedBy References IsSourceOf IsDerivedFrom "+
  "IsNewVersionOf IsPreviousVersionOf").split())
@@ -1737,6 +1773,11 @@ def _moderate_reference(self, ref, doval=True):
  except AttributeError as ex:
  raise InvalidUpdate("location or proxyFor: value is not a string", sys=self) from ex
 
+ # Penultimately, add an id if doesn't already have one
+ if not ref.get("@id"):
+ ref['@id'] = "REPLACE"
+ # ref['@id'] = random_id("ref:")
+
  # Finally, validate (if requested)
  schemauri = NERDM_SCH_ID + "/definitions/BibliographicReference"
  if ref.get("_schema"):
@@ -1747,6 +1788,8 @@ def _moderate_reference(self, ref, doval=True):
  if doval:
  self.validate_json(ref, schemauri)
 
+ if ref.get("@id") == "REPLACE":
+ del ref['@id']
  return ref
 
  def _moderate_file(self, cmp, doval=True):
@@ -1980,6 +2023,8 @@ def __init__(self, dbcli_factory: DBClientFactory, log: Logger, config: dict={},
 class DAPProjectDataHandler(ProjectDataHandler):
  """
  A :py:class:`~nistoar.midas.wsgi.project.ProjectDataHandler` specialized for editing NERDm records.
+
+ Note that this implementation inherits its PUT, PATCH, and DELETE handling from its super-class.
  """
  _allowed_post_paths = "authors references components".split() + [FILE_DELIM, LINK_DELIM]
 

python/nistoar/midas/dap/service/validate.py

@@ -1,12 +1,45 @@
 """
 validation utilities specialized for DAP editing
 """
+import re
+
 from nistoar.nerdm.validate import *
 from nistoar.nerdm.constants import core_schema_base as CORE_SCHEMA_BASE
+import nistoar.nerdm.utils as nerdm_utils
 
 PUB_SCHEMA_BASE = CORE_SCHEMA_BASE + "pub/"
+RLS_SCHEMA_BASE = CORE_SCHEMA_BASE + "rls/"
+
+directives_by_uribase = {
+ CORE_SCHEMA_BASE: {
+ "derequire": [ "Resource", "Organization" ]
+ },
+ PUB_SCHEMA_BASE: {
+ "derequire": [ "PublicDataResource", "Person" ]
+ },
+ RLS_SCHEMA_BASE: {
+ "derequire": [ "ReleasedResource" ]
+ }
+}
+_verre = re.compile(r"/v\d.*$")
 
 class LenientSchemaLoader(ejs.SchemaLoader):
+ """
+ this modifies the schema definitions on selected schemas to be more lenient for records 
+ intended for use in the DAP Authoring API.
+ """
+ def load_schema(self, uri):
+ out = super().load_schema(uri)
+
+ if out.get("id", "").startswith(CORE_SCHEMA_BASE):
+ base = _verre.sub("/", out['id'])
+ directives = directives_by_uribase.get(base, {})
+ nerdm_utils.loosen_schema(out, directives)
+
+ return out
+
+
+class OldLenientSchemaLoader(ejs.SchemaLoader):
  """
  this modifies the schema definitions on selected schemas to be more lenient for records 
  intended for use in the DAP Authoring API.

python/nistoar/midas/dbio/project.py

@@ -142,6 +142,14 @@ def create_record(self, name, data=None, meta=None) -> ProjectRecord:
  self.log.info("Created %s record %s (%s) for %s", self.dbcli.project, prec.id, prec.name, self.who)
  return prec
 
+ def delete_record(self, id) -> ProjectRecord:
+ """
+ delete the draft record. This may leave a stub record in place if, for example, the record 
+ has been published previously. 
+ """
+ # TODO: handling previously published records
+ raise NotImplementedError()
+
  def _get_id_shoulder(self, user: PubAgent):
  """
  return an ID shoulder that is appropriate for the given user agent
@@ -231,7 +239,7 @@ def update_data(self, id, newdata, part=None, message="", _prec=None):
  """
  merge the given data into the currently save data content for the record with the given identifier.
  :param str id: the identifier for the record whose data should be updated.
- :param str newdata: the data to save as the new content. 
+ :param str|dict|list newdata: the data to save as the new content. 
  :param str part: the slash-delimited pointer to an internal data property. If provided, 
  the given ``newdata`` is a value that should be set to the property pointed 
  to by ``part``. 
@@ -498,13 +506,16 @@ def _save_data(self, indata: Mapping, prec: ProjectRecord,
  def _validate_data(self, data):
  pass
 
- def clear_data(self, id: str, part: str=None, message: str=None, prec=None):
+ def clear_data(self, id: str, part: str=None, message: str=None, prec=None) -> bool:
  """
- remove the stored data content of the record and reset it to its defaults. 
+ remove the stored data content of the record and reset it to its defaults. Note that
+ no change is recorded if the requested data does not exist yet.
  :param str id: the identifier for the record whose data should be cleared.
  :param stt part: the slash-delimited pointer to an internal data property. If provided, 
  only that property will be cleared (either removed or set to an initial
  default).
+ :return: True the data was properly cleared; return False if ``part`` was specified but does not
+ yet exist in the data.
  :param ProjectRecord prec: the previously fetched and possibly updated record corresponding to `id`.
  If this is not provided, the record will by fetched anew based on the `id`. 
  :raises ObjectNotFound: if no record with the given ID exists or the `part` parameter points to 
@@ -518,7 +529,7 @@ def clear_data(self, id: str, part: str=None, message: str=None, prec=None):
  set_state = True
  prec = self.dbcli.get_record_for(id, ACLs.WRITE) # may raise ObjectNotFound/NotAuthorized
 
- if _prec.status.state not in [status.EDIT, status.READY]:
+ if prec.status.state not in [status.EDIT, status.READY]:
  raise NotEditable(id)
 
  initdata = self._new_data_for(prec.id, prec.meta)
@@ -541,7 +552,7 @@ def clear_data(self, id: str, part: str=None, message: str=None, prec=None):
  elif prop not in data:
  data[prop] = {}
  elif prop not in data:
- break
+ return False
  elif not steps:
  del data[prop]
  break
@@ -576,7 +587,8 @@ def clear_data(self, id: str, part: str=None, message: str=None, prec=None):
  finally:
  self._record_action(provact)
  self.log.info("Cleared out data for %s record %s (%s) for %s",
- self.dbcli.project, _prec.id, _prec.name, self.who)
+ self.dbcli.project, prec.id, prec.name, self.who)
+ return True
 
 
  def update_status_message(self, id: str, message: str, _prec=None) -> status.RecordStatus: