List bookmarks API on Dashboard

metadata_service/api/swagger_doc/template.yml

@@ -258,6 +258,9 @@ components:
  name:
  type: string
  description: 'Dashboard name'
+ url:
+ type: string
+ description: 'Dashboard URL'
  description:
  type: string
  description: 'Dashboard description'
@@ -305,6 +308,33 @@ components:
  description: 'Badges associated with the Dashboard'
  items:
  $ref: '#/components/schemas/TagFields'
+ DashboardSummary:
+ type: object
+ properties:
+ uri:
+ type: string
+ description: 'Unique identifier of the dashboard'
+ cluster:
+ type: string
+ description: 'Cluster name'
+ group_name:
+ type: string
+ description: 'Dashboard group name'
+ group_url:
+ type: string
+ description: 'Dashboard group URL'
+ name:
+ type: string
+ description: 'Dashboard name'
+ url:
+ type: string
+ description: 'Dashboard URL'
+ description:
+ type: string
+ description: 'Dashboard description'
+ last_successful_run_timestamp:
+ type: int
+ description: 'Dashboard last run timestamp in epoch'
  ErrorResponse:
  type: object
  properties:

metadata_service/api/user.py

@@ -1,18 +1,19 @@
 import logging
 from http import HTTPStatus
-from typing import Iterable, Mapping, Optional, Union
-from flask import current_app as app
+from typing import Iterable, Mapping, Optional, Union, Dict, List, Any # noqa: F401
 
+from amundsen_common.models.dashboard import DashboardSummarySchema
 from amundsen_common.models.popular_table import PopularTableSchema
 from amundsen_common.models.user import UserSchema
 from flasgger import swag_from
+from flask import current_app as app
 from flask_restful import Resource
 
 from metadata_service.api import BaseAPI
+from metadata_service.entity.resource_type import to_resource_type, ResourceType
 from metadata_service.exception import NotFoundException
 from metadata_service.proxy import get_proxy_client
 from metadata_service.util import UserResourceRel
-from metadata_service.entity.resource_type import to_resource_type
 
 LOGGER = logging.getLogger(__name__)
 
@@ -57,9 +58,24 @@ def get(self, user_id: str) -> Iterable[Union[Mapping, int, None]]:
  try:
  resources = self.client.get_table_by_user_relation(user_email=user_id,
  relation_type=UserResourceRel.follow)
- if len(resources['table']) > 0:
- return {'table': PopularTableSchema(many=True).dump(resources['table']).data}, HTTPStatus.OK
- return {'table': []}, HTTPStatus.OK
+
+ table_key = ResourceType.Table.name.lower()
+ dashboard_key = ResourceType.Dashboard.name.lower()
+ result = {
+ table_key: [],
+ dashboard_key: []
+ } # type: Dict[str, List[Any]]
+
+ if table_key in resources and len(resources[table_key]) > 0:
+ result[table_key] = PopularTableSchema(many=True).dump(resources[table_key]).data
+
+ resources = self.client.get_dashboard_by_user_relation(user_email=user_id,
+ relation_type=UserResourceRel.follow)
+
+ if dashboard_key in resources and len(resources[dashboard_key]) > 0:
+ result[dashboard_key] = DashboardSummarySchema(many=True).dump(resources[dashboard_key]).data
+
+ return result, HTTPStatus.OK
 
  except NotFoundException:
  return {'message': 'user_id {} does not exist'.format(user_id)}, HTTPStatus.NOT_FOUND
@@ -102,8 +118,7 @@ def put(self, user_id: str, resource_type: str, resource_id: str) -> Iterable[Un
  return {'message': 'The user {} for id {} resource type {}'
  'is not added successfully'.format(user_id,
  resource_id,
- resource_type)}, \
- HTTPStatus.INTERNAL_SERVER_ERROR
+ resource_type)}, HTTPStatus.INTERNAL_SERVER_ERROR
 
  @swag_from('swagger_doc/user/follow_delete.yml')
  def delete(self, user_id: str, resource_type: str, resource_id: str) -> Iterable[Union[Mapping, int, None]]:
@@ -128,8 +143,7 @@ def delete(self, user_id: str, resource_type: str, resource_id: str) -> Iterable
  return {'message': 'The user {} for id {} resource type {} '
  'is not deleted successfully'.format(user_id,
  resource_id,
- resource_type)}, \
- HTTPStatus.INTERNAL_SERVER_ERROR
+ resource_type)}, HTTPStatus.INTERNAL_SERVER_ERROR
 
 
 class UserOwnsAPI(Resource):

metadata_service/entity/dashboard_detail.py

@@ -20,6 +20,7 @@ class DashboardDetail:
  description: Optional[str] = attr.ib()
  created_timestamp: Optional[int] = attr.ib()
  updated_timestamp: Optional[int] = attr.ib()
+ last_successful_run_timestamp: Optional[int] = attr.ib()
  last_run_timestamp: Optional[int] = attr.ib()
  last_run_state: Optional[str] = attr.ib()
  owners: List[User] = attr.ib(factory=list)

metadata_service/proxy/atlas_proxy.py

@@ -6,6 +6,7 @@
 from amundsen_common.models.popular_table import PopularTable
 from amundsen_common.models.table import Column, Statistics, Table, Tag, User
 from amundsen_common.models.user import User as UserEntity
+from amundsen_common.models.dashboard import DashboardSummary
 from atlasclient.client import Atlas
 from atlasclient.exceptions import BadRequest
 from atlasclient.models import EntityUniqueAttribute
@@ -526,6 +527,10 @@ def get_tags(self) -> List:
  )
  return tags
 
+ def get_dashboard_by_user_relation(self, *, user_email: str, relation_type: UserResourceRel) \
+ -> Dict[str, List[DashboardSummary]]:
+ pass
+
  def get_table_by_user_relation(self, *, user_email: str, relation_type: UserResourceRel) -> Dict[str, Any]:
  params = {
  'typeName': self.READER_TYPE,

metadata_service/proxy/base_proxy.py

@@ -4,6 +4,7 @@
 from amundsen_common.models.popular_table import PopularTable
 from amundsen_common.models.table import Table
 from amundsen_common.models.user import User as UserEntity
+from amundsen_common.models.dashboard import DashboardSummary
 
 from metadata_service.entity.dashboard_detail import DashboardDetail as DashboardDetailEntity
 from metadata_service.entity.description import Description
@@ -81,6 +82,11 @@ def get_latest_updated_ts(self) -> int:
  def get_tags(self) -> List:
  pass
 
+ @abstractmethod
+ def get_dashboard_by_user_relation(self, *, user_email: str, relation_type: UserResourceRel) \
+ -> Dict[str, List[DashboardSummary]]:
+ pass
+
  @abstractmethod
  def get_table_by_user_relation(self, *, user_email: str,
  relation_type: UserResourceRel) -> Dict[str, Any]:

metadata_service/proxy/gremlin_proxy.py

@@ -6,6 +6,7 @@
 from amundsen_common.models.popular_table import PopularTable
 from amundsen_common.models.table import Table
 from amundsen_common.models.user import User as UserEntity
+from amundsen_common.models.dashboard import DashboardSummary
 from gremlin_python.driver.driver_remote_connection import \
  DriverRemoteConnection
 from gremlin_python.process.anonymous_traversal import traversal
@@ -140,6 +141,10 @@ def get_latest_updated_ts(self) -> int:
  def get_tags(self) -> List:
  pass
 
+ def get_dashboard_by_user_relation(self, *, user_email: str, relation_type: UserResourceRel) \
+ -> Dict[str, List[DashboardSummary]]:
+ pass
+
  def get_table_by_user_relation(self, *, user_email: str,
  relation_type: UserResourceRel) -> Dict[str, Any]:
  pass

metadata_service/proxy/neo4j_proxy.py

@@ -5,6 +5,7 @@
 from typing import (Any, Dict, List, Optional, Tuple, Union, # noqa: F401
  no_type_check)
 
+from amundsen_common.models.dashboard import DashboardSummary
 from amundsen_common.models.popular_table import PopularTable
 from amundsen_common.models.table import (Application, Column, Reader, Source,
  Statistics, Table, User,
@@ -853,57 +854,75 @@ def _get_user_resource_relationship_clause(relation_type: UserResourceRel, id: s
  raise NotImplementedError(f'The relation type {relation_type} is not defined!')
  return relation
 
- @staticmethod
- def _get_user_table_relationship_clause(relation_type: UserResourceRel, tbl_key: str = None,
- user_key: str = None) -> str:
- """
- Returns the relationship clause of a cypher query between users and tables
- The User node is 'usr', the table node is 'tbl', and the relationship is 'rel'
- e.g. (usr:User)-[rel:READ]->(tbl:Table), (usr)-[rel:READ]->(tbl)
+ @timer_with_counter
+ def get_dashboard_by_user_relation(self, *, user_email: str, relation_type: UserResourceRel) \
+ -> Dict[str, List[DashboardSummary]]:
  """
- tbl_matcher: str = ''
- user_matcher: str = ''
+ Retrieve all follow the Dashboard per user based on the relation.
 
- if tbl_key is not None:
- tbl_matcher += ':Table'
- if tbl_key != '':
- tbl_matcher += f' {{key: "{tbl_key}"}}'
+ :param user_email: the email of the user
+ :param relation_type: the relation between the user and the resource
+ :return:
+ """
+ rel_clause: str = self._get_user_resource_relationship_clause(relation_type=relation_type,
+ id='',
+ resource_type=ResourceType.Dashboard,
+ user_key=user_email)
 
- if user_key is not None:
- user_matcher += ':User'
- if user_key != '':
- user_matcher += f' {{key: "{user_key}"}}'
+ query = textwrap.dedent(f"""
+ MATCH {rel_clause}<-[:DASHBOARD]-(dg:Dashboardgroup)<-[:DASHBOARD_GROUP]-(clstr:Cluster)
+ OPTIONAL MATCH (resource)-[:DESCRIPTION]->(dscrpt:Description)
+ OPTIONAL MATCH (resource)-[:EXECUTED]->(last_exec:Execution)
+ WHERE split(last_exec.key, '/')[5] = '_last_successful_execution'
+ RETURN clstr.name as cluster_name, dg.name as dg_name, dg.dashboard_group_url as dg_url,
+ resource.key as uri, resource.name as name, resource.dashboard_url as url,
+ dscrpt.description as description, last_exec.timestamp as last_successful_run_timestamp""")
+
+ records = self._execute_cypher_query(statement=query, param_dict={'user_key': user_email})
+
+ if not records:
+ raise NotFoundException('User {user_id} does not {relation} on {resource_type} resources'.format(
+ user_id=user_email,
+ relation=relation_type,
+ resource_type=ResourceType.Dashboard.name))
 
- if relation_type == UserResourceRel.follow:
- relation = f'(usr{user_matcher})-[rel:FOLLOW]->(tbl{tbl_matcher})'
- elif relation_type == UserResourceRel.own:
- relation = f'(usr{user_matcher})<-[rel:OWNER]-(tbl{tbl_matcher})'
- elif relation_type == UserResourceRel.read:
- relation = f'(usr{user_matcher})-[rel:READ]->(tbl{tbl_matcher})'
- else:
- raise NotImplementedError(f'The relation type {relation_type} is not defined!')
- return relation
+ results = []
+ for record in records:
+ results.append(DashboardSummary(
+ uri=record['uri'],
+ cluster=record['cluster_name'],
+ group_name=record['dg_name'],
+ group_url=record['dg_url'],
+ name=record['name'],
+ url=record['url'],
+ description=record['description'],
+ last_successful_run_timestamp=record['last_successful_run_timestamp'],
+ ))
+
+ return {ResourceType.Dashboard.name.lower(): results}
 
  @timer_with_counter
- def get_table_by_user_relation(self, *, user_email: str, relation_type: UserResourceRel) -> Dict[str, Any]:
+ def get_table_by_user_relation(self, *, user_email: str, relation_type: UserResourceRel) \
+ -> Dict[str, List[PopularTable]]:
  """
- Retrive all follow the resources per user based on the relation.
- We start with table resources only, then add dashboard.
+ Retrive all follow the Table per user based on the relation.
 
  :param user_email: the email of the user
  :param relation_type: the relation between the user and the resource
  :return:
  """
- rel_clause: str = self._get_user_table_relationship_clause(relation_type=relation_type,
- tbl_key='',
- user_key=user_email)
+ rel_clause: str = self._get_user_resource_relationship_clause(relation_type=relation_type,
+ id='',
+ resource_type=ResourceType.Table,
+ user_key=user_email)
+
  query = textwrap.dedent(f"""
- MATCH {rel_clause}<-[:TABLE]-(schema:Schema)<-[:SCHEMA]-(clstr:Cluster)<-[:CLUSTER]-(db:Database)
- WITH db, clstr, schema, tbl
- OPTIONAL MATCH (tbl)-[:DESCRIPTION]->(tbl_dscrpt:Description)
- RETURN db, clstr, schema, tbl, tbl_dscrpt""")
+  MATCH {rel_clause}<-[:TABLE]-(schema:Schema)<-[:SCHEMA]-(clstr:Cluster)<-[:CLUSTER]-(db:Database)
+  WITH db, clstr, schema, resource
+  OPTIONAL MATCH (resource)-[:DESCRIPTION]->(tbl_dscrpt:Description)
+  RETURN db, clstr, schema, resource, tbl_dscrpt""")
 
- table_records = self._execute_cypher_query(statement=query, param_dict={'query_key': user_email})
+ table_records = self._execute_cypher_query(statement=query, param_dict={'user_key': user_email})
 
  if not table_records:
  raise NotFoundException('User {user_id} does not {relation} any resources'.format(user_id=user_email,
@@ -914,9 +933,9 @@ def get_table_by_user_relation(self, *, user_email: str, relation_type: UserReso
  database=record['db']['name'],
  cluster=record['clstr']['name'],
  schema=record['schema']['name'],
- name=record['tbl']['name'],
+ name=record['resource']['name'],
  description=self._safe_get(record, 'tbl_dscrpt', 'description')))
- return {'table': results}
+ return {ResourceType.Table.name.lower(): results}
 
  @timer_with_counter
  def get_frequently_used_tables(self, *, user_email: str) -> Dict[str, Any]:
@@ -1048,6 +1067,8 @@ def get_dashboard(self,
  MATCH (d:Dashboard {key: $query_key})-[:DASHBOARD_OF]->(dg:Dashboardgroup)-[:DASHBOARD_GROUP_OF]->(c:Cluster)
  OPTIONAL MATCH (d)-[:DESCRIPTION]->(description:Description)
  OPTIONAL MATCH (d)-[:EXECUTED]->(last_exec:Execution) WHERE split(last_exec.key, '/')[5] = '_last_execution'
+ OPTIONAL MATCH (d)-[:EXECUTED]->(last_success_exec:Execution)
+ WHERE split(last_success_exec.key, '/')[5] = '_last_successful_execution'
  OPTIONAL MATCH (d)-[:LAST_UPDATED_AT]->(t:Timestamp)
  OPTIONAL MATCH (d)-[:OWNER]->(owner:User)
  OPTIONAL MATCH (d)-[:TAG]->(tag:Tag)
@@ -1060,6 +1081,7 @@ def get_dashboard(self,
  description.description as description,
  dg.name as group_name,
  dg.dashboard_group_url as group_url,
+ toInteger(last_success_exec.timestamp) as last_successful_run_timestamp,
  toInteger(last_exec.timestamp) as last_run_timestamp,
  last_exec.state as last_run_state,
  toInteger(t.timestamp) as updated_timestamp,
@@ -1084,6 +1106,8 @@ def get_dashboard(self,
  description=self._safe_get(record, 'description'),
  group_name=self._safe_get(record, 'group_name'),
  group_url=self._safe_get(record, 'group_url'),
+ last_successful_run_timestamp=self._safe_get(record,
+ 'last_successful_run_timestamp'),
  last_run_timestamp=self._safe_get(record, 'last_run_timestamp'),
  last_run_state=self._safe_get(record, 'last_run_state'),
  updated_timestamp=self._safe_get(record, 'updated_timestamp'),

requirements.txt

@@ -41,7 +41,7 @@ pytest-mock==1.1
 typing==3.6.4
 
 
-amundsen-common==0.2.6
+amundsen-common>=0.3.0,<1.0
 flasgger==0.9.3
 Flask-RESTful==0.3.6
 Flask==1.0.2

setup.py

@@ -2,7 +2,7 @@
 
 from setuptools import setup, find_packages
 
-__version__ = '2.4.0'
+__version__ = '2.4.1rc0'
 
 
 requirements_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'requirements.txt')

tests/unit/api/test_user.py

@@ -46,6 +46,8 @@ def setUp(self, mock_get_proxy_client: MagicMock) -> None:
 
  def test_get(self) -> None:
  self.mock_client.get_table_by_user_relation.return_value = {'table': []}
+ self.mock_client.get_dashboard_by_user_relation.return_value = {'dashboard': []}
+
  response = self.api.get(user_id='username')
  self.assertEqual(list(response)[1], HTTPStatus.OK)
  self.mock_client.get_table_by_user_relation.assert_called_once()