API Docs for TM-Admin¶

tmadmin_manage.py¶

dbsupport.py¶

DBSupport ¶

DBSupport(table)

Bases: object

Parameters:

Name	Type	Description	Default
`table`	`str`	The table to use for this connection.	required

Returns:

Type	Description
`DBSupport`	An instance of this class

Source code in tm_admin/dbsupport.py

def __init__(self,
             table: str,
            ):
    """
    A base class since all tables have the same structure for queries.

    Args:
        table (str): The table to use for this connection.

    Returns:
        (DBSupport): An instance of this class
    """
    self.pg = None
    self.table = table
    self.columns = None

connect `async` ¶

connect(dburi='localhost/tm_admin')

Parameters:

Name	Type	Description	Default
`dburi`	`str`	The URI string for the database connection.	`'localhost/tm_admin'`

Source code in tm_admin/dbsupport.py

async def connect(self,
                dburi: str = "localhost/tm_admin",
                ):
    """
    Args:
        dburi (str): The URI string for the database connection.
    """
    profile = f"{self.table.capitalize()}Table()"
    self.profile = eval(profile)
    if dburi:
        self.pg = PostgresClient()
        await self.pg.connect(dburi)
    self.types = dir(tm_admin.types_tm)

createTable `async` ¶

createTable(obj)

Create a table in a postgres database.

Parameters:

Name	Type	Description	Default
`obj`		The config data for the table.	required

Source code in tm_admin/dbsupport.py

async def createTable(self,
                obj,
                ):
    """
    Create a table in a postgres database.

    Args:
        obj: The config data for the table.
    """
    sql = f"INSERT INTO {self.table}(id, "
    for column,value in obj.data.items():
        # print(f"{column} is {type(value)}")
        if type(value) == str:
            # FIXME: for now ignore timestamps, as they're meaningless
            # between projects
            try:
                if parse(value):
                    continue
            except:
                # it's a string, but not a date
                pass
        if value is not None:
            sql += f"{column},"
    sql = sql[:-1]
    sql += f") VALUES("
    for column,value in obj.data.items():
        try:
            if parse(value):
                continue
        except:
            pass
        if column == 'id':
            sql += f"nextval('public.{self.table}_id_seq'),"
            continue
        if value is None:
            continue
        elif type(value) == datetime:
            continue
        elif type(value) == int:
            sql += f"{value},"
        elif type(value) == bool:
            if value:
                sql += f"true,"
            else:
                sql += f"false,"
        elif type(value) == str:
            sql += f"'{value}',"

    #print(sql[:-1])
    result = await self.pg.execute(f"{sql[:-1]});")

updateTable `async` ¶

updateTable(id=None)

Updates an existing table in the database

Parameters:

Name	Type	Description	Default
`id`	`int`	The ID of the dataset to update	`None`

Source code in tm_admin/dbsupport.py

async def updateTable(self,
                id: int = None,
                ):
    """
    Updates an existing table in the database

    Args:
        id (int): The ID of the dataset to update
    """
    sql = f"UPDATE {self.table} SET"
    if not id:
        id = profile.data['id']
    for column,value in self.profile.data.items():
        name = column.replace('_', '').capitalize()
        if name in self.types:
            # FIXME: this needs to not be hardcoded!
            tmp = tm_admin.types_tm.Mappinglevel._member_names_
            if type(value) == str:
                level = value
            else:
                level = tmp[value-1]
            sql += f" {column}='{level}'"
            continue
        if value:
            try:
                # FIXME: for now ignore timestamps, as they're meaningless
                # between projects
                if parse(value):
                    continue
            except:
                # it's a string, but not a date
                pass
            sql += f" {column}='{value}',"
    sql += f" WHERE id='{id}'"
    # print(sql)
    result = await self.pg.execute(f"{sql[:-1]}';")

resetSequence `async` ¶

resetSequence()

Reset the postgres sequence to zero.

Source code in tm_admin/dbsupport.py

async def resetSequence(self):
    """
    Reset the postgres sequence to zero.
    """
    sql = f"ALTER SEQUENCE public.{self.table}_id_seq RESTART;"
    await self.pg.execute(sql)

getByID `async` ¶

getByID(id)

Return the data for the ID in the table.

Parameters:

Name	Type	Description	Default
`id`	`int`	The ID of the dataset to retrieve.	required

Returns:

Type	Description
`dict`	The results of the query

Source code in tm_admin/dbsupport.py

async def getByID(self,
            id: int,
            ):
    """
    Return the data for the ID in the table.

    Args:
        id (int): The ID of the dataset to retrieve.

    Returns:
        (dict): The results of the query
    """

    data = await self.getByWhere(f" id={id}")
    if len(data) == 0:
        return dict()
    else:
        return data[0][0]

getByName `async` ¶

getByName(name)

Return the data for the name in the table.

Parameters:

Name	Type	Description	Default
`name`	`str`	The name of the dataset to retrieve.	required

Returns:

Type	Description
`list`	The results of the query

Source code in tm_admin/dbsupport.py

async def getByName(self,
            name: str,
            ):
    """
    Return the data for the name in the table.

    Args:
        name (str): The name of the dataset to retrieve.

    Returns:
        (list): The results of the query
    """
    data = await self.getByWhere(f" name='{name}'")

    if len(data) == 0:
        return dict()
    else:
        return data[0][0]

getAll `async` ¶

getAll()

Return all the data in the table.

Returns:

Type	Description
`list`	The results of the query

Source code in tm_admin/dbsupport.py

async def getAll(self):
    """
    Return all the data in the table.

    Returns:
        (list): The results of the query
    """
    sql = f"SELECT row_to_json({self.table}) as row FROM {self.table}"
    # print(sql)
    result = list()
    result = await self.pg.execute(sql)

    return result

getByWhere `async` ¶

getByWhere(where)

Return the data for the where clause in the table.

Parameters:

Name	Type	Description	Default
`where`	`str`	The where clause of the dataset to retrieve.	required

Returns:

Type	Description
`list`	The results of the query

Source code in tm_admin/dbsupport.py

async def getByWhere(self,
            where: str,
            ):
    """
    Return the data for the where clause in the table.

    Args:
        where (str): The where clause of the dataset to retrieve.

    Returns:
        (list): The results of the query
    """
    sql = f"SELECT row_to_json({self.table}) as row FROM {self.table} WHERE {where}"
    # print(sql)
    result = await self.pg.execute(sql)

    return result

getByLocation `async` ¶

getByLocation(location, table='projects')

Return the database records in a table using GPS coordinates.

Parameters:

Name	Type	Description	Default
`location`	`Point`	The location to use to find the project or task.	required

Returns:

Type	Description
`list`	The results of the query

Source code in tm_admin/dbsupport.py

async def getByLocation(self,
            location: Point,
            table: str = 'projects',
            ):
    """
    Return the database records in a table using GPS coordinates.

    Args:
        location (Point): The location to use to find the project or task.

    Returns:
        (list): The results of the query
    """
    data = dict()
    ewkt = shape(location)
    sql = f"SELECT row_to_json({self.table}) as row FROM {table} WHERE ST_CONTAINS(ST_GeomFromEWKT('SRID=4326;{ewkt}') geom)"
    result = await self.pg.execute(sql)

    return result

deleteByID `async` ¶

deleteByID(id)

Delete the record for the ID in the table.

Parameters:

Name	Type	Description	Default
`id`	`int`	The ID of the dataset to delete.	required

Source code in tm_admin/dbsupport.py

async def deleteByID(self,
            id: int,
            ):
    """
    Delete the record for the ID in the table.

    Args:
        id (int): The ID of the dataset to delete.
    """
    sql = f"DELETE FROM {self.table} WHERE id='{id}'"
    result = self.pg.execute(sql)
    return True

getColumn `async` ¶

getColumn(uid, column)

This gets a single column from the database.

Parameters:

Name	Type	Description	Default
`uid`	`int`	The ID to get	required
`column`	`str`	The column.	required

Returns:

Type	Description
`list`	The column values

Source code in tm_admin/dbsupport.py

async def getColumn(self,
             uid: int,
             column: str,
             ):
    """
    This gets a single column from the database.

    Args:
        uid (int): The ID to get
        column (str): The column.

    Returns:
        (list): The column values
    """
    sql = f"SELECT {column} FROM {self.table} WHERE id={uid}"
    result = await self.pg.execute(sql)

    if len(result) > 0:
        return result[0][column]
    else:
        return None

updateColumn `async` ¶

updateColumn(uid, data)

This updates a single column in the database. If you want to update multiple columns, use self.updateTable() instead.

Parameters:

Name	Type	Description	Default
`uid`	`int`	The ID of the user to update	required
`data`	`dict`	The column and new value	required

Source code in tm_admin/dbsupport.py

async def updateColumn(self,
                uid: int,
                data: dict,
                ):
    """
    This updates a single column in the database. If you want to update multiple columns,
    use self.updateTable() instead.

    Args:
        uid (int): The ID of the user to update
        data (dict): The column and new value
    """
    [[column, value]] = data.items()
    sql = f"UPDATE {self.table} SET {column}='{value}' WHERE id='{uid}'"
    # print(sql)
    await self.pg.execute(f"{sql};")

    return True

removeColumn `async` ¶

removeColumn(uid, data)

This updates a single array column in the database. If you want to update multiple columns, use self.updateTable() instead.

Parameters:

Name	Type	Description	Default
`uid`	`int`	The ID of the user to update	required
`data`	`dict`	The column and new value	required

Source code in tm_admin/dbsupport.py

async def removeColumn(self,
                uid: int,
                data: dict,
                ):
    """
    This updates a single array column in the database.
    If you want to update multiple columns, use self.updateTable()
    instead.

    Args:
        uid (int): The ID of the user to update
        data (dict): The column and new value
    """
    [[column, value]] = data.items()
    aval = "'{" + f"{value}" + "}"
    sql = f"UPDATE {self.table} SET {column}=array_remove({column}, {value}) WHERE id='{uid}'"
    # print(sql)
    result = await self.pg.execute(f"{sql};")

appendColumn `async` ¶

appendColumn(uid, data)

This updates a single array column in the database. If you want to update multiple columns, use self.updateTable() instead.

Parameters:

Name	Type	Description	Default
`uid`	`int`	The ID of the user to update	required
`data`	`dict`	The column and new value	required

Source code in tm_admin/dbsupport.py

async def appendColumn(self,
                uid: int,
                data: dict,
                ):
    """
    This updates a single array column in the database.
    If you want to update multiple columns, use self.updateTable()
    instead.

    Args:
        uid (int): The ID of the user to update
        data (dict): The column and new value
    """
    [[column, value]] = data.items()
    aval = "'{" + f"{value}" + "}"
    sql = f"UPDATE {self.table} SET {column}={column}||{aval}' WHERE id='{uid}'"
    #print(sql)
    result = await self.pg.execute(f"{sql};")

renameTable `async` ¶

renameTable(table)

Source code in tm_admin/dbsupport.py

async def renameTable(self,
                    table: str,
                    ):
    """
    """
    sql = f"DROP TABLE IF EXISTS {table}_bak"
    result = await self.pg.execute(sql)
    sql = f"ALTER TABLE {table} RENAME TO {table}_bak;"
    result = await self.pg.execute(sql)
    sql = f"ALTER TABLE new_{table} RENAME TO {table};"
    result = await self.pg.execute(sql)
    sql = f"DROP TABLE IF EXISTS {table}_bak CASCADE"
    result = await self.pg.execute(sql)

    print(f"renameTable{self.pg.dburi}")
    # These are copied for the TM4 database, but have been merged
    # into the local database so JOIN works faster than remote
    # access, or looping through tons of data in Python.
    sql = f"DROP TABLE IF EXISTS user_interests CASCADE"
    result = await self.pg.execute(sql)
    sql = f"DROP TABLE IF EXISTS user_licenses CASCADE"
    result = await self.pg.execute(sql)
    sql = f"DROP TABLE IF EXISTS team_members CASCADE"
    result = await self.pg.execute(sql)

copyTable `async` ¶

copyTable(table, remote)

Use DBLINK to copy a table from the Tasking Manager database to a local table so JOINing is much faster.

Parameters:

Name	Type	Description	Default
`table`	`str`	The table to copy	required

Source code in tm_admin/dbsupport.py

async def copyTable(self,
                    table: str,
                    remote: PostgresClient,
                    ):
    """
    Use DBLINK to copy a table from the Tasking Manager
    database to a local table so JOINing is much faster.

    Args:
        table (str): The table to copy
    """
    timer = Timer(initial_text=f"Copying {table}...",
                  text="copying {table} took {seconds:.0f}s",
                  logger=log.debug,
                )
    # Get the columns from the remote database table
    self.columns = await remote.getColumns(table)

    print(f"SELF: {self.pg.dburi}")
    print(f"REMOTE: {remote.dburi}")

    # Do we already have a local copy ?
    sql = f"SELECT FROM pg_tables WHERE schemaname = 'public' AND tablename  = '{table}'"
    result = await self.pg.execute(sql)
    print(result)

    # cleanup old temporary tables in the current database
    # drop = ["DROP TABLE IF EXISTS users_bak",
    #         "DROP TABLE IF EXISTS user_interests",
    #         "DROP TABLE IF EXISTS foo"]
    # result = await pg.pg.executemany(drop)
    sql = f"DROP TABLE IF EXISTS new_{table} CASCADE"
    result = await self.pg.execute(sql)
    sql = f"DROP TABLE IF EXISTS {table}_bak CASCADE"
    result = await self.pg.execute(sql)
    timer.start()
    dbuser = self.pg.dburi["dbuser"]
    dbpass = self.pg.dburi["dbpass"]
    sql = f"CREATE SERVER IF NOT EXISTS pg_rep_db FOREIGN DATA WRAPPER dblink_fdw  OPTIONS (dbname 'tm4');"
    data = await self.pg.execute(sql)

    sql = f"CREATE USER MAPPING IF NOT EXISTS FOR {dbuser} SERVER pg_rep_db OPTIONS ( user '{dbuser}', password '{dbpass}');"
    result = await self.pg.execute(sql)

    # Copy table from remote database so JOIN is faster when it's in the
    # same database
    #columns = await sel.getColumns(table)
    log.warning(f"Copying a remote table is slow, but faster than remote access......")
    sql = f"SELECT * INTO {table} FROM dblink('pg_rep_db','SELECT * FROM {table}') AS {table}({self.columns})"
    print(sql)
    result = await self.pg.execute(sql)

    return True

main `async` ¶

main()

This main function lets this class be run standalone by a bash script.

Source code in tm_admin/dbsupport.py

async def main():
    """This main function lets this class be run standalone by a bash script."""
    parser = argparse.ArgumentParser()
    parser.add_argument("-v", "--verbose", nargs="?", const="0", help="verbose output")
    parser.add_argument("-u", "--uri", default='localhost/tm_admin',
                            help="Database URI")
    # parser.add_argument("-r", "--reset", help="Reset Sequences")
    args = parser.parse_args()

    # if len(argv) <= 1:
    #     parser.print_help()
    #     quit()

    # if verbose, dump to the terminal.
    log_level = os.getenv("LOG_LEVEL", default="INFO")
    if args.verbose is not None:
        log_level = logging.DEBUG

    logging.basicConfig(
        level=log_level,
        format=("%(asctime)s.%(msecs)03d [%(levelname)s] " "%(name)s | %(funcName)s:%(lineno)d | %(message)s"),
        datefmt="%y-%m-%d %H:%M:%S",
        stream=sys.stdout,
    )

    org = DBSupport('organizations')
    await org.connect(args.uri)
    # organization.resetSequence()
    all = await org.getAll()

    # Don't pass id, let postgres auto increment
    ut = OrganizationsTable(name='test org', slug="slug", type=1)
#                            orgtype=tm_admin.types_tm.Organizationtype.FREE)
    await org.createTable(ut)
    # print(all)

    all = await org.getByID(1)
    print(all)

    all = await org.getByName('fixme')
    print(all)

options: show_source: false heading_level: 3

pgsupport.py¶

DBSupport ¶

DBSupport(table)

Bases: object

Parameters:

Name	Type	Description	Default
`table`	`str`	The table to use for this connection.	required

Returns:

Type	Description
`DBSupport`	An instance of this class

Source code in tm_admin/dbsupport.py

def __init__(self,
             table: str,
            ):
    """
    A base class since all tables have the same structure for queries.

    Args:
        table (str): The table to use for this connection.

    Returns:
        (DBSupport): An instance of this class
    """
    self.pg = None
    self.table = table
    self.columns = None

connect `async` ¶

connect(dburi='localhost/tm_admin')

Parameters:

Name	Type	Description	Default
`dburi`	`str`	The URI string for the database connection.	`'localhost/tm_admin'`

Source code in tm_admin/dbsupport.py

async def connect(self,
                dburi: str = "localhost/tm_admin",
                ):
    """
    Args:
        dburi (str): The URI string for the database connection.
    """
    profile = f"{self.table.capitalize()}Table()"
    self.profile = eval(profile)
    if dburi:
        self.pg = PostgresClient()
        await self.pg.connect(dburi)
    self.types = dir(tm_admin.types_tm)

createTable `async` ¶

createTable(obj)

Create a table in a postgres database.

Parameters:

Name	Type	Description	Default
`obj`		The config data for the table.	required

Source code in tm_admin/dbsupport.py

async def createTable(self,
                obj,
                ):
    """
    Create a table in a postgres database.

    Args:
        obj: The config data for the table.
    """
    sql = f"INSERT INTO {self.table}(id, "
    for column,value in obj.data.items():
        # print(f"{column} is {type(value)}")
        if type(value) == str:
            # FIXME: for now ignore timestamps, as they're meaningless
            # between projects
            try:
                if parse(value):
                    continue
            except:
                # it's a string, but not a date
                pass
        if value is not None:
            sql += f"{column},"
    sql = sql[:-1]
    sql += f") VALUES("
    for column,value in obj.data.items():
        try:
            if parse(value):
                continue
        except:
            pass
        if column == 'id':
            sql += f"nextval('public.{self.table}_id_seq'),"
            continue
        if value is None:
            continue
        elif type(value) == datetime:
            continue
        elif type(value) == int:
            sql += f"{value},"
        elif type(value) == bool:
            if value:
                sql += f"true,"
            else:
                sql += f"false,"
        elif type(value) == str:
            sql += f"'{value}',"

    #print(sql[:-1])
    result = await self.pg.execute(f"{sql[:-1]});")

updateTable `async` ¶

updateTable(id=None)

Updates an existing table in the database

Parameters:

Name	Type	Description	Default
`id`	`int`	The ID of the dataset to update	`None`

Source code in tm_admin/dbsupport.py

async def updateTable(self,
                id: int = None,
                ):
    """
    Updates an existing table in the database

    Args:
        id (int): The ID of the dataset to update
    """
    sql = f"UPDATE {self.table} SET"
    if not id:
        id = profile.data['id']
    for column,value in self.profile.data.items():
        name = column.replace('_', '').capitalize()
        if name in self.types:
            # FIXME: this needs to not be hardcoded!
            tmp = tm_admin.types_tm.Mappinglevel._member_names_
            if type(value) == str:
                level = value
            else:
                level = tmp[value-1]
            sql += f" {column}='{level}'"
            continue
        if value:
            try:
                # FIXME: for now ignore timestamps, as they're meaningless
                # between projects
                if parse(value):
                    continue
            except:
                # it's a string, but not a date
                pass
            sql += f" {column}='{value}',"
    sql += f" WHERE id='{id}'"
    # print(sql)
    result = await self.pg.execute(f"{sql[:-1]}';")

resetSequence `async` ¶

resetSequence()

Reset the postgres sequence to zero.

Source code in tm_admin/dbsupport.py

async def resetSequence(self):
    """
    Reset the postgres sequence to zero.
    """
    sql = f"ALTER SEQUENCE public.{self.table}_id_seq RESTART;"
    await self.pg.execute(sql)

getByID `async` ¶

getByID(id)

Return the data for the ID in the table.

Parameters:

Name	Type	Description	Default
`id`	`int`	The ID of the dataset to retrieve.	required

Returns:

Type	Description
`dict`	The results of the query

Source code in tm_admin/dbsupport.py

async def getByID(self,
            id: int,
            ):
    """
    Return the data for the ID in the table.

    Args:
        id (int): The ID of the dataset to retrieve.

    Returns:
        (dict): The results of the query
    """

    data = await self.getByWhere(f" id={id}")
    if len(data) == 0:
        return dict()
    else:
        return data[0][0]

getByName `async` ¶

getByName(name)

Return the data for the name in the table.

Parameters:

Name	Type	Description	Default
`name`	`str`	The name of the dataset to retrieve.	required

Returns:

Type	Description
`list`	The results of the query

Source code in tm_admin/dbsupport.py

async def getByName(self,
            name: str,
            ):
    """
    Return the data for the name in the table.

    Args:
        name (str): The name of the dataset to retrieve.

    Returns:
        (list): The results of the query
    """
    data = await self.getByWhere(f" name='{name}'")

    if len(data) == 0:
        return dict()
    else:
        return data[0][0]

getAll `async` ¶

getAll()

Return all the data in the table.

Returns:

Type	Description
`list`	The results of the query

Source code in tm_admin/dbsupport.py

async def getAll(self):
    """
    Return all the data in the table.

    Returns:
        (list): The results of the query
    """
    sql = f"SELECT row_to_json({self.table}) as row FROM {self.table}"
    # print(sql)
    result = list()
    result = await self.pg.execute(sql)

    return result

getByWhere `async` ¶

getByWhere(where)

Return the data for the where clause in the table.

Parameters:

Name	Type	Description	Default
`where`	`str`	The where clause of the dataset to retrieve.	required

Returns:

Type	Description
`list`	The results of the query

Source code in tm_admin/dbsupport.py

async def getByWhere(self,
            where: str,
            ):
    """
    Return the data for the where clause in the table.

    Args:
        where (str): The where clause of the dataset to retrieve.

    Returns:
        (list): The results of the query
    """
    sql = f"SELECT row_to_json({self.table}) as row FROM {self.table} WHERE {where}"
    # print(sql)
    result = await self.pg.execute(sql)

    return result

getByLocation `async` ¶

getByLocation(location, table='projects')

Return the database records in a table using GPS coordinates.

Parameters:

Name	Type	Description	Default
`location`	`Point`	The location to use to find the project or task.	required

Returns:

Type	Description
`list`	The results of the query

Source code in tm_admin/dbsupport.py

async def getByLocation(self,
            location: Point,
            table: str = 'projects',
            ):
    """
    Return the database records in a table using GPS coordinates.

    Args:
        location (Point): The location to use to find the project or task.

    Returns:
        (list): The results of the query
    """
    data = dict()
    ewkt = shape(location)
    sql = f"SELECT row_to_json({self.table}) as row FROM {table} WHERE ST_CONTAINS(ST_GeomFromEWKT('SRID=4326;{ewkt}') geom)"
    result = await self.pg.execute(sql)

    return result

deleteByID `async` ¶

deleteByID(id)

Delete the record for the ID in the table.

Parameters:

Name	Type	Description	Default
`id`	`int`	The ID of the dataset to delete.	required

Source code in tm_admin/dbsupport.py

async def deleteByID(self,
            id: int,
            ):
    """
    Delete the record for the ID in the table.

    Args:
        id (int): The ID of the dataset to delete.
    """
    sql = f"DELETE FROM {self.table} WHERE id='{id}'"
    result = self.pg.execute(sql)
    return True

getColumn `async` ¶

getColumn(uid, column)

This gets a single column from the database.

Parameters:

Name	Type	Description	Default
`uid`	`int`	The ID to get	required
`column`	`str`	The column.	required

Returns:

Type	Description
`list`	The column values

Source code in tm_admin/dbsupport.py

async def getColumn(self,
             uid: int,
             column: str,
             ):
    """
    This gets a single column from the database.

    Args:
        uid (int): The ID to get
        column (str): The column.

    Returns:
        (list): The column values
    """
    sql = f"SELECT {column} FROM {self.table} WHERE id={uid}"
    result = await self.pg.execute(sql)

    if len(result) > 0:
        return result[0][column]
    else:
        return None

updateColumn `async` ¶

updateColumn(uid, data)

This updates a single column in the database. If you want to update multiple columns, use self.updateTable() instead.

Parameters:

Name	Type	Description	Default
`uid`	`int`	The ID of the user to update	required
`data`	`dict`	The column and new value	required

Source code in tm_admin/dbsupport.py

async def updateColumn(self,
                uid: int,
                data: dict,
                ):
    """
    This updates a single column in the database. If you want to update multiple columns,
    use self.updateTable() instead.

    Args:
        uid (int): The ID of the user to update
        data (dict): The column and new value
    """
    [[column, value]] = data.items()
    sql = f"UPDATE {self.table} SET {column}='{value}' WHERE id='{uid}'"
    # print(sql)
    await self.pg.execute(f"{sql};")

    return True

removeColumn `async` ¶

removeColumn(uid, data)

This updates a single array column in the database. If you want to update multiple columns, use self.updateTable() instead.

Parameters:

Name	Type	Description	Default
`uid`	`int`	The ID of the user to update	required
`data`	`dict`	The column and new value	required

Source code in tm_admin/dbsupport.py

async def removeColumn(self,
                uid: int,
                data: dict,
                ):
    """
    This updates a single array column in the database.
    If you want to update multiple columns, use self.updateTable()
    instead.

    Args:
        uid (int): The ID of the user to update
        data (dict): The column and new value
    """
    [[column, value]] = data.items()
    aval = "'{" + f"{value}" + "}"
    sql = f"UPDATE {self.table} SET {column}=array_remove({column}, {value}) WHERE id='{uid}'"
    # print(sql)
    result = await self.pg.execute(f"{sql};")

appendColumn `async` ¶

appendColumn(uid, data)

This updates a single array column in the database. If you want to update multiple columns, use self.updateTable() instead.

Parameters:

Name	Type	Description	Default
`uid`	`int`	The ID of the user to update	required
`data`	`dict`	The column and new value	required

Source code in tm_admin/dbsupport.py

async def appendColumn(self,
                uid: int,
                data: dict,
                ):
    """
    This updates a single array column in the database.
    If you want to update multiple columns, use self.updateTable()
    instead.

    Args:
        uid (int): The ID of the user to update
        data (dict): The column and new value
    """
    [[column, value]] = data.items()
    aval = "'{" + f"{value}" + "}"
    sql = f"UPDATE {self.table} SET {column}={column}||{aval}' WHERE id='{uid}'"
    #print(sql)
    result = await self.pg.execute(f"{sql};")

renameTable `async` ¶

renameTable(table)

Source code in tm_admin/dbsupport.py

async def renameTable(self,
                    table: str,
                    ):
    """
    """
    sql = f"DROP TABLE IF EXISTS {table}_bak"
    result = await self.pg.execute(sql)
    sql = f"ALTER TABLE {table} RENAME TO {table}_bak;"
    result = await self.pg.execute(sql)
    sql = f"ALTER TABLE new_{table} RENAME TO {table};"
    result = await self.pg.execute(sql)
    sql = f"DROP TABLE IF EXISTS {table}_bak CASCADE"
    result = await self.pg.execute(sql)

    print(f"renameTable{self.pg.dburi}")
    # These are copied for the TM4 database, but have been merged
    # into the local database so JOIN works faster than remote
    # access, or looping through tons of data in Python.
    sql = f"DROP TABLE IF EXISTS user_interests CASCADE"
    result = await self.pg.execute(sql)
    sql = f"DROP TABLE IF EXISTS user_licenses CASCADE"
    result = await self.pg.execute(sql)
    sql = f"DROP TABLE IF EXISTS team_members CASCADE"
    result = await self.pg.execute(sql)

copyTable `async` ¶

copyTable(table, remote)

Use DBLINK to copy a table from the Tasking Manager database to a local table so JOINing is much faster.

Parameters:

Name	Type	Description	Default
`table`	`str`	The table to copy	required

Source code in tm_admin/dbsupport.py

async def copyTable(self,
                    table: str,
                    remote: PostgresClient,
                    ):
    """
    Use DBLINK to copy a table from the Tasking Manager
    database to a local table so JOINing is much faster.

    Args:
        table (str): The table to copy
    """
    timer = Timer(initial_text=f"Copying {table}...",
                  text="copying {table} took {seconds:.0f}s",
                  logger=log.debug,
                )
    # Get the columns from the remote database table
    self.columns = await remote.getColumns(table)

    print(f"SELF: {self.pg.dburi}")
    print(f"REMOTE: {remote.dburi}")

    # Do we already have a local copy ?
    sql = f"SELECT FROM pg_tables WHERE schemaname = 'public' AND tablename  = '{table}'"
    result = await self.pg.execute(sql)
    print(result)

    # cleanup old temporary tables in the current database
    # drop = ["DROP TABLE IF EXISTS users_bak",
    #         "DROP TABLE IF EXISTS user_interests",
    #         "DROP TABLE IF EXISTS foo"]
    # result = await pg.pg.executemany(drop)
    sql = f"DROP TABLE IF EXISTS new_{table} CASCADE"
    result = await self.pg.execute(sql)
    sql = f"DROP TABLE IF EXISTS {table}_bak CASCADE"
    result = await self.pg.execute(sql)
    timer.start()
    dbuser = self.pg.dburi["dbuser"]
    dbpass = self.pg.dburi["dbpass"]
    sql = f"CREATE SERVER IF NOT EXISTS pg_rep_db FOREIGN DATA WRAPPER dblink_fdw  OPTIONS (dbname 'tm4');"
    data = await self.pg.execute(sql)

    sql = f"CREATE USER MAPPING IF NOT EXISTS FOR {dbuser} SERVER pg_rep_db OPTIONS ( user '{dbuser}', password '{dbpass}');"
    result = await self.pg.execute(sql)

    # Copy table from remote database so JOIN is faster when it's in the
    # same database
    #columns = await sel.getColumns(table)
    log.warning(f"Copying a remote table is slow, but faster than remote access......")
    sql = f"SELECT * INTO {table} FROM dblink('pg_rep_db','SELECT * FROM {table}') AS {table}({self.columns})"
    print(sql)
    result = await self.pg.execute(sql)

    return True

main `async` ¶

main()

This main function lets this class be run standalone by a bash script.

Source code in tm_admin/dbsupport.py

async def main():
    """This main function lets this class be run standalone by a bash script."""
    parser = argparse.ArgumentParser()
    parser.add_argument("-v", "--verbose", nargs="?", const="0", help="verbose output")
    parser.add_argument("-u", "--uri", default='localhost/tm_admin',
                            help="Database URI")
    # parser.add_argument("-r", "--reset", help="Reset Sequences")
    args = parser.parse_args()

    # if len(argv) <= 1:
    #     parser.print_help()
    #     quit()

    # if verbose, dump to the terminal.
    log_level = os.getenv("LOG_LEVEL", default="INFO")
    if args.verbose is not None:
        log_level = logging.DEBUG

    logging.basicConfig(
        level=log_level,
        format=("%(asctime)s.%(msecs)03d [%(levelname)s] " "%(name)s | %(funcName)s:%(lineno)d | %(message)s"),
        datefmt="%y-%m-%d %H:%M:%S",
        stream=sys.stdout,
    )

    org = DBSupport('organizations')
    await org.connect(args.uri)
    # organization.resetSequence()
    all = await org.getAll()

    # Don't pass id, let postgres auto increment
    ut = OrganizationsTable(name='test org', slug="slug", type=1)
#                            orgtype=tm_admin.types_tm.Organizationtype.FREE)
    await org.createTable(ut)
    # print(all)

    all = await org.getByID(1)
    print(all)

    all = await org.getByName('fixme')
    print(all)

options: show_source: false heading_level: 3

generator.py¶

Generator ¶

Generator(filespec=None)

Bases: object

Parameters:

Name	Type	Description	Default
`filespec`	`str`	The config file to use as source.	`None`

Returns:

Type	Description
`Generator`	An instance of this class

Source code in tm_admin/generator.py

def __init__(self,
            filespec: str = None,
            ):
    """
    A class that generates the output files from the config data.

    Args:
        filespec (str): The config file to use as source.

    Returns:
        (Generator): An instance of this class
    """
    self.filespec = None
    self.yaml = None
    if filespec:
        self.filespec = Path(filespec)
        self.yaml = YamlFile(filespec)
    self.createTypes()
    self.yaml2py = {'int32': 'int',
                'int64': 'int',
                'bool': 'bool',
                'string': 'str',
                'bytes': 'bytes',
                'timestamp': 'timestamp without time zone',
                'polygon': 'Polygon',
                'point': 'Point',
                'jsonb': 'dict',
                }

    self.yaml2sql = {'int32': 'int',
                'int64': 'bigint',
                'bool': 'bool',
                'string': 'character varying',
                'bytes': 'bytea',
                'timestamp': 'timestamp without time zone',
                'polygon': 'geometry(Polygon,4326)',
                'point': 'geometry(Point,4326)',
                'jsonb': 'jsonb',
                }

readConfig ¶

readConfig(filespec)

Reads in the YAML config file.

Parameters:

Name	Type	Description	Default
`filespec`	`str`	The config file to use as source.	required

Source code in tm_admin/generator.py

def readConfig(self,
                filespec: str,
                ):
    """
    Reads in the YAML config file.

    Args:
        filespec (str): The config file to use as source.
    """
    self.filespec = Path(filespec)
    self.yaml = YamlFile(filespec)

createTypes ¶

createTypes()

Creates the enum files, which need to be done first, since the other generated files reference these.

Source code in tm_admin/generator.py

def createTypes(self):
    """
    Creates the enum files, which need to be done first, since the
    other generated files reference these.
    """
    gen = self.readConfig(f'{rootdir}/types.yaml')
    out = self.createSQLEnums()
    with open('types_tm.sql', 'w') as file:
        file.write(out)
        file.close()
    out = self.createProtoEnums()
    with open('types_tm.proto', 'w') as file:
        file.write(out)
        file.close()
    out = self.createPyEnums()
    with open('types_tm.py', 'w') as file:
        file.write(out)
        file.close()

createSQLEnums ¶

createSQLEnums()

Create an input file for postgres of the custom types.

Returns:

Type	Description
`str`	The source for postgres to create the SQL types.

Source code in tm_admin/generator.py

def createSQLEnums(self):
    """
    Create an input file for postgres of the custom types.

    Returns:
        (str): The source for postgres to create the SQL types.
    """
    out = ""
    for entry in self.yaml.yaml:
        [[table, values]] = entry.items()
        out += f"DROP TYPE IF EXISTS public.{table} CASCADE;\n"
        out += f"CREATE TYPE public.{table} AS ENUM (\n"
        for line in values:
            out += f"\t'{line}',\n"
        out = out[:-2]
        out += "\n);\n"
    return out

createProtoEnums ¶

createProtoEnums()

Create an input file for postgres of the custom types.

Returns:

Type	Description
`str`	The source for protoc to create the Protobuf types.

Source code in tm_admin/generator.py

def createProtoEnums(self):
    """
    Create an input file for postgres of the custom types.

    Returns:
        (str): The source for protoc to create the Protobuf types.
    """
    out = "syntax = 'proto3';\n\n"
    for entry in self.yaml.yaml:
        index = 0
        [[table, values]] = entry.items()
        out += f"enum {table.capitalize()} {{\n"
        for line in values:
            out += f"\t{line} = {index};\n"
            index += 1
        out += "};\n\n"
    return out

createPyEnums ¶

createPyEnums()

Create an input file for python of the custom types.

Returns:

Type	Description
`str`	The source for python to create the enums.

Source code in tm_admin/generator.py

def createPyEnums(self):
    """
    Create an input file for python of the custom types.

    Returns:
        (str): The source for python to create the enums.
    """
    out = f"import logging\n"
    out += f"from enum import IntEnum\n"
    for entry in self.yaml.yaml:
        index = 1
        [[table, values]] = entry.items()
        out += f"class {table.capitalize()}(IntEnum):\n"
        for line in values:
            out += f"\t{line} = {index}\n"
            index += 1
        out += '\n'
    return out

createPyMessage ¶

createPyMessage()

Creates a python class wrapper for protobuf.

Returns:

Type	Description
`str`	The source for python to create the class stubs.

Source code in tm_admin/generator.py

    def createPyMessage(self):
        """
        Creates a python class wrapper for protobuf.

        Returns:
            (str): The source for python to create the class stubs.
        """
        out = f"""
import logging
from datetime import timedelta
# from shapely.geometry import Polygon, Point, shape

log = logging.getLogger(__name__)
        """
        for entry in self.yaml.yaml:
            [[table, settings]] = entry.items()
            out += f"""
class {table.capitalize()}Message(object):
    def __init__(self, 
            """
            # print(table, settings)
            share = False
            datatype = None
            data = "        self.data = {"
            for item in settings:
                if type(item) == dict:
                    [[k, v]] = item.items()
                    # print(v)
                    if v == "jsonb":
                        datatype = "dict"
                        breakpoint()
                    for k1 in v:
                        if type(k1) == dict:
                            [[k2, v2]] = k1.items()
                            if k2 == 'share' and v2:
                                share = True
                        elif type(k1) == str:
                            if k1 in self.yaml2py:
                                datatype = self.yaml2py[k1]
                            else:
                                datatype = item
                                continue
                    if share:
                        share = False
                        out += f"{k}: {datatype} = None, "
                        data += f"'{k}': {k}, "
        out += "):\n"
        out += f"{data[:-2]}}}\n"

        return out

createPyClass ¶

createPyClass()

Creates a python class wrapper for the protobuf messages.

Returns:

Type	Description
`str`	The source for python to create the class stubs.

Source code in tm_admin/generator.py

    def createPyClass(self):
        """
        Creates a python class wrapper for the protobuf messages.

        Returns:
            (str): The source for python to create the class stubs.
        """
        out = f"""
import logging
from datetime import datetime
import tm_admin.types_tm
from shapely.geometry import Point, LineString, Polygon

log = logging.getLogger(__name__)

        """
        for entry in self.yaml.yaml:
            [[table, settings]] = entry.items()
            out += f"""
class {table.capitalize()}Table(object):
    def __init__(self, 
            """
            datatype = None
            now = datetime.now()
            data = "            self.data = {"
            for item in settings:
                if type(item) == dict:
                    [[k, v]] = item.items()
                    for k1 in v:
                        if type(k1) == dict:
                            continue
                        elif type(k1) == str:
                            if k1[:15] == 'public.geometry':
                                datatype = k1[16:-1].split(',')[0]
                                log.warning(f"GEOMETRY: {datatype}")
                            elif k1[:7] == 'public.':
                                # FIXME: It's in the SQL types
                                datatype = f"tm_admin.types_tm.{k1[7:].capitalize()}"
                                # log.warning(f"SQL ENUM {k1}! {datatype}")
                            elif k1 in self.yaml2py:
                                datatype = self.yaml2py[k1]
                            else:
                                datatype = item
                                continue
                        if k1 == 'bool':
                            out += f"{k}: {datatype} = False, "
                        elif k1 == 'timestamp':
                            out += f"{k}: datetime = '{datetime.now()}', "
                        elif k1[:7] == 'public.':
                            defined = f"tm_admin.types_tm.{k1[7:].capitalize()}"
                            # log.warning(f"SQL ENUM {k1}!!")
                            default = eval(f"{defined}(1)")
                            out += f"{k}: {defined} =  {defined}.{default.name}, "
                            # out += f"{k}: int =  1, "
                        else:
                            out += f"{k}: {datatype} = None, "
                        # print(k)
                        data += f"'{k}': {k}, "
            out = out[:-2]
            out += "):\n"
            out += f"{data[:-2]}}}\n"

        return out

createProtoMessage ¶

createProtoMessage()

Create the source for a protobuf message

Returns:

Type	Description
`list`	The protobuf message source.

Source code in tm_admin/generator.py

def createProtoMessage(self):
    """
    Create the source for a protobuf message

    Returns:
        (list): The protobuf message source.
    """
    pb = ProtoBuf()
    out = pb.createTableProto(self.yaml.yaml)
    return out

createSQLTable ¶

createSQLTable()

Create the source for an SQL table.

Returns:

Type	Description
`str`	The protobuf message source.

Source code in tm_admin/generator.py

    def createSQLTable(self):
        """
        Create the source for an SQL table.

        Returns:
            (str): The protobuf message source.
        """
        out = "-- Do not edit this file, it's generated from the yaml file\n\n"
        sequence = list()
        primary = list()
        partition = list()
        for entry in self.yaml.yaml:
            [[table, values]] = entry.items()
            out += f"DROP TABLE IF EXISTS public.{table} CASCADE;\n"
            out += f"CREATE TABLE public.{table} (\n"
            unique = list()
            typedef = ""
            for line in values:
                # these are usually from the types.yaml file, which have no
                # settings beyond the enum value.
                if type(line) == str:
                    # print(f"SQL TABLE: {typedef} {line}")
                    typedef = table
                    continue
                [[k, v]] = line.items()
                required = ""
                array = ""
                public = False
                primary = list()
                partition = list()
                for item in v:
                    if type(item) == dict:
                        if 'sequence' in item and item['sequence']:
                            sequence.append(k)
                        if 'required' in item and item['required']:
                            required = ' NOT NULL'
                        if 'array' in item and item['array']:
                            array = "[]"
                        if 'unique' in item and item['unique']:
                            unique.append(k)
                        if 'primary' in item and item['primary']:
                            primary.append(k)
                        if 'partition' in item and item['partition']:
                            partition.append(k)
                    if len(v) >= 2:
                        if 'required' in v[1] and v[1]['required']:
                            required = ' NOT NULL'
                    if type(item) == str:
                        if item[:7] == 'public.' and item[15:8] != 'geometry':
                            public = True
                        # elif item[15:8] == 'geometry':
                        #     out += f"\t{k} {self.yaml2py[v[0]]}{array}{required},\n"
                if public:
                    out += f"\t{k} {v[0]}{array}{required},\n"
                else:
                    # print(v)
                    # FIXME: if this produces an error, check the yaml file as this
                    # usually means the type field isn't first in the list.
                    if v[0][:6] == 'table.':
                        out += f"\t{k} {v[0][6:]}{array},\n"
                    else:
                        try:
                            out += f"\t{k} {self.yaml2sql[v[0]]}{array}{required},\n"
                        except:
                            breakpoint()
            if len(unique) > 0:
                keys = str(unique).replace("'", "")[1:-1];
                out += f"\tUNIQUE({keys})\n);\n"
            if out[-2:] == ',\n':
                out = f"{out[:-2]}\n);\n\n"
            if len(sequence) > 0:
                for key in sequence:
                    out += f"""
DROP SEQUENCE IF EXISTS public.{table}_{key}_seq CASCADE;
CREATE SEQUENCE public.{table}_{key}_seq
        START WITH 1
        INCREMENT BY 1
        NO MINVALUE
        NO MAXVALUE
        CACHE 1;

"""
        if len(primary) > 0:
            keys = str(primary).replace("'", "")[1:-1];
            out += f"\tALTER TABLE {table} ADD CONSTRAINT {table}_pkey PRIMARY KEY({keys})\n);\n"

        return out + "\n"

main ¶

main()

This main function lets this class be run standalone by a bash script.

Source code in tm_admin/generator.py

def main():
    """This main function lets this class be run standalone by a bash script."""
    parser = argparse.ArgumentParser(
        prog="generator",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description="Generate SQL, Protobuf, and Python data structures",
        epilog="""
        This should only be run standalone for debugging purposes.
        """,
    )
    parser.add_argument("-v", "--verbose", nargs="?", const="0", help="verbose output")
    args, known = parser.parse_known_args()

    if len(argv) <= 1:
        parser.print_help()
        quit()

    # if verbose, dump to the terminal.
    log_level = os.getenv("LOG_LEVEL", default="INFO")
    if args.verbose is not None:
        log_level = logging.DEBUG

    logging.basicConfig(
        level=log_level,
        format=("%(asctime)s.%(msecs)03d [%(levelname)s] " "%(name)s | %(funcName)s:%(lineno)d | %(message)s"),
        datefmt="%y-%m-%d %H:%M:%S",
        stream=sys.stdout,
    )

    gen = Generator()
    for config in known:
        gen.readConfig(config)
        out = gen.createSQLTable()
        sqlfile = config.replace('.yaml', '.sql')
        path = Path(sqlfile)
        #if path.exists():
        #    path.rename(file.replace('.sql', '_bak.sql'))
        with open(sqlfile, 'w') as file:
            file.write(out)
            log.info(f"Wrote {sqlfile} to disk")
            file.close()
        proto = config.replace('.yaml', '.proto')
        # out = gen.createProtoMessage()
        # with open(proto, 'w') as file:
        #     file.writelines([str(i)+'\n' for i in out])
        #     log.info(f"Wrote {proto} to disk")
        #     file.close()

        print(config)
        out = gen.createPyClass()
        py = config.replace('.yaml', '_class.py')
        with open(py, 'w') as file:
            file.write(out)
            log.info(f"Wrote {py} to disk")
            file.close()
        # print(out)
        out = gen.createPyMessage()
        py = config.replace('.yaml', '_proto.py')
        with open(py, 'w') as file:
            file.write(out)
            log.info(f"Wrote {py} to disk")
            file.close()

options: show_source: false heading_level: 3

proto.py¶

ProtoBuf ¶

ProtoBuf(sqlfile=None)

Bases: object

Returns:

Type	Description
`ProtoBuf`	An instance of this class

Source code in tm_admin/proto.py

def __init__(self,
            sqlfile: str = None,
            ):
    """
    A class that generates protobuf files from the config data.

    Returns:
        (ProtoBuf): An instance of this class
    """
    self.sqlfile = sqlfile

createEnumProto ¶

createEnumProto(enums)

Process a list of enums into the protobuf version.

Parameters:

Name	Type	Description	Default
`enums`	`dict`	The list of tables to generate a protobuf for.	required

Returns:

Type	Description
`list`	The list of enums in protobuf format

Source code in tm_admin/proto.py

def createEnumProto(self,
                enums: dict,
                ):
    """
    Process a list of enums into the protobuf version.

    Args:
        enums (dict): The list of tables to generate a protobuf for.

    Returns:
        (list): The list of enums in protobuf format
    """
    out = list()
    out.append(f"syntax = 'proto3';")
    for name, value in enums.items():
        index = 0
        out.append(f"enum {name.capitalize()} {{")
        for entry in value:
            out.append(f"\t{entry} = {index};")
            index += 1
        out.append('};')

    return out

createTableProto ¶

createTableProto(tables)

Process a list of tables into the protobuf version.

Parameters:

Name	Type	Description	Default
`tables`	`list`	The list of tables to generate a protobuf for.	required

Returns:

Type	Description
`list`	The list of tables in protobuf format

Source code in tm_admin/proto.py

def createTableProto(self,
                tables: list,
                ):
    """
    Process a list of tables into the protobuf version.

    Args:
        tables (list): The list of tables to generate a protobuf for.

    Returns:
        (list): The list of tables in protobuf format
    """
    out = list()
    out.append(f"syntax = 'proto3';")
    # types.proto is generated from the types.yaml file.
    # out.append("import 'types_tm.proto';")
    out.append("package tmadmin;")
    out.append("import 'types_tm.proto';")
    out.append("import 'google/protobuf/timestamp.proto';")

    convert = {'timestamp': "google.protobuf.Timestamp",
               'polygon': 'bytes', 'point': 'bytes'}
    for table in tables:
        index = 1
        for key, value in table.items():
            out.append(f"message {key} {{")
            optional = ""
            repeated = ""
            # print(f"VALUE: {value}")
            for data in value:
                #if type(data) == str:
                #    log.warning(f"ENUM: {data}")
                #    continue
                for entry, settings in data.items():
                    # print(f"DATA: {entry} = {settings}")
                    #    datatype = settings[0][7:].capitalize()
                    share = False
                    array = ""
                    datatype = None
                    required = ""
                    optional = ""
                    for item in settings:
                        if type(item) == str:
                            # print(f"DATA: {item}")
                            if item[:15] == 'public.geometry':
                                datatype = "bytes"
                            elif item[:7] == 'public.':
                                datatype = item[7:].capitalize()
                            elif item in convert:
                                datatype = convert[item]
                            else:
                                datatype = item
                            continue
                        if type(item) == dict:
                            [[k, v]] = item.items()
                            if k == 'required' and v:
                                required = k
                            if k == 'optional' and v:
                                optional = k
                            if k == 'share':
                                share = True
                            if k == 'array':
                                array = "repeated"
                    if not share:
                        continue
                    # out.append(f"\t{required} {optional} {datatype} {entry} = {index};")
                    out.append(f"\t {array} {optional} {datatype} {entry} = {index};")
                    index += 1
        out.append(f"}}")

    return out

main ¶

main()

This main function lets this class be run standalone by a bash script.

Source code in tm_admin/proto.py

def main():
    """This main function lets this class be run standalone by a bash script."""
    parser = argparse.ArgumentParser(
        prog="config",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description="Manage the postgres database for the tm-admin project",
        epilog="""
        This should only be run standalone for debugging purposes.
        """,
    )
    parser.add_argument("-v", "--verbose", nargs="?", const="0", help="verbose output")
    args, known = parser.parse_known_args()

    if len(argv) <= 1:
        parser.print_help()
        # quit()

    # if verbose, dump to the terminal.
    log_level = os.getenv("LOG_LEVEL", default="INFO")
    if args.verbose is not None:
        log_level = logging.DEBUG

    logging.basicConfig(
        level=log_level,
        format=("%(asctime)s.%(msecs)03d [%(levelname)s] " "%(name)s | %(funcName)s:%(lineno)d | %(message)s"),
        datefmt="%y-%m-%d %H:%M:%S",
        stream=sys.stdout,
    )

    tm = ProtoBuf()
    for table in known:
        out1, out2 = tm.createProtoFromSQL(table)
        name = table.replace('.sql', '.proto')
        # pyfile = table.replace('.sql', '.py')
        # xx = tm.protoToDict(name)
        # name = table.replace('.yaml', '.proto')
        out = tm.createTableProto()
        if len(out1) > 0:
            with open(name, 'w') as file:
                file.writelines([str(i)+'\n' for i in out1])
                file.close()
        if len(out2) > 0:
            with open(name, 'w') as file:
                file.writelines([str(i)+'\n' for i in out2])
                file.close()
        log.info(f"Wrote {name} to disk")

options: show_source: false heading_level: 3

tmserver.py¶

TMServer ¶

TMServer(target)

Bases: object

Parameters:

Name	Type	Description	Default
`target`	`str`	The name of the target program	required

Returns:

Type	Description
`TMServer`	An instance of this class

Source code in tm_admin/tmserver.py

def __init__(self,
             target: str,
             ):
    """
    Instantiate a server

    Args:
        target (str): The name of the target program

    Returns:
        (TMServer): An instance of this class
    """
    self.hosts = YamlFile(f"{rootdir}/services.yaml")
    log.debug("Starting server")
    server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
    tm_admin.services_pb2_grpc.add_TMAdminServicer_to_server(
        RequestServicer(), server
    )
    # Enable reflection for grpc_cli
    SERVICE_NAMES = (
        tm_admin.services_pb2.DESCRIPTOR.services_by_name['TMAdmin'].full_name,
        reflection.SERVICE_NAME,
    )
    reflection.enable_server_reflection(SERVICE_NAMES, server)

    target = self.getTarget(target)
    [[host, port]] = target.items()
    # FIXME: this needs to use SSL for a secure connection
    server.add_insecure_port(f"[::]:{port}")
    server.start()
    server.wait_for_termination()

getTarget ¶

getTarget(target)

Get the target hostname and IP port number

Parameters:

Name	Type	Description	Default
`target`	`str`	The name of the target program	required

Returns:

Type	Description
`dict`	the hostname and IP port for this target program

Source code in tm_admin/tmserver.py

def getTarget(self,
            target: str,
            ):
    """
    Get the target hostname and IP port number

    Args:
        target (str): The name of the target program

    Returns:
        (dict): the hostname and IP port for this target program
    """
    return self.hosts.yaml[0][target][0]

main ¶

main()

This main function lets this class be run standalone by a bash script.

Source code in tm_admin/tmserver.py

def main():
    """This main function lets this class be run standalone by a bash script."""
    parser = argparse.ArgumentParser(
        prog="tmserver",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description="Server for gRPC communication",
        epilog="""
        This should only be run standalone for debugging purposes.
        """,
    )
    parser.add_argument("-v", "--verbose", nargs="?", const="0", help="verbose output")

    args = parser.parse_args()

    # if len(argv) <= 1:
    #     parser.print_help()
    #     quit()

    # if verbose, dump to the terminal.
    log_level = os.getenv("LOG_LEVEL", default="INFO")
    if args.verbose is not None:
        log_level = logging.DEBUG

    logging.basicConfig(
        level=log_level,
        format=("%(asctime)s.%(msecs)03d [%(levelname)s] " "%(name)s | %(funcName)s:%(lineno)d | %(message)s"),
        datefmt="%y-%m-%d %H:%M:%S",
        stream=sys.stdout,
    )

    # This blocks till  this process is killed
    tm = TMServer('test')

options: show_source: false heading_level: 3

tmclient.py¶

TMClient ¶

TMClient(target)

Bases: object

Parameters:

Name	Type	Description	Default
`target`	`str`	The name of the target program	required

Returns:

Type	Description
`TMClient`	An instance of this class

Source code in tm_admin/tmclient.py

def __init__(self,
            target: str,
            ):
    """
    Instantiate a client

    Args:
        target (str): The name of the target program

    Returns:
        (TMClient): An instance of this class
    """
    # the services.yaml file defines the hostname and ports for all programs.
    self.hosts = YamlFile(f"{rootdir}/services.yaml")
    target = self.getTarget(target)
    [[host, port]] = target.items()
    # FIXME: this needs to use SSL for a secure connection
    self.channel = grpc.insecure_channel(f"{host}:{port}")
    # self.stub = tm_admin.services_pb2_grpc.TMClientStub(channel)
    # self.stub = tm_admin.services_pb2_grpc.testStub(self.channel)
    self.stub = tm_admin.services_pb2_grpc.TMAdminStub(self.channel)

sendUserProfile ¶

sendUserProfile(msg)

Send data to the target program

Parameters:

Name	Type	Description	Default
`msg`	`str`	The message to send	required

Returns:

Type	Description
`dict`	The response from the server

Source code in tm_admin/tmclient.py

def sendUserProfile(self,
            msg: str,
            ):
    """
    Send data to the target program

    Args:
        msg (str): The message to send

    Returns:
       (dict): The response from the server
    """
    foo = UsersMessage(id=1, username=msg, name=msg)

    bar = serialize_to_protobuf(foo.data, tm_admin.users.users_pb2.users)

    response = self.stub.GetUser(bar)
    #print(f"TMAdmin client received: {response}")
    return response

getTarget ¶

getTarget(target)

Get the target hostname and IP port number

Parameters:

Name	Type	Description	Default
`target`	`str`	The name of the target program	required

Returns:

Type	Description
`dict`	the hostname and IP port for this target program

Source code in tm_admin/tmclient.py

def getTarget(self,
            target: str,
            ):
    """
    Get the target hostname and IP port number

    Args:
        target (str): The name of the target program

    Returns:
        (dict): the hostname and IP port for this target program
    """
    return self.hosts.yaml[0][target][0]

main ¶

main()

This main function lets this class be run standalone by a bash script.

Source code in tm_admin/tmclient.py

def main():
    """This main function lets this class be run standalone by a bash script."""
    parser = argparse.ArgumentParser(
        prog="tmclient",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description="gRPC Client",
        epilog="""
        This should only be run standalone for debugging purposes.
        """,
    )
    parser.add_argument("-v", "--verbose", nargs="?", const="0", help="verbose output")
    parser.add_argument("-m", "--msg", default='who', help="string to send")
    args, known = parser.parse_known_args()

    # if len(argv) <= 1:
    #     parser.print_help()
    #     quit()

    # if verbose, dump to the terminal.
    log_level = os.getenv("LOG_LEVEL", default="INFO")
    if args.verbose is not None:
        log_level = logging.DEBUG

    logging.basicConfig(
        level=log_level,
        format=("%(asctime)s.%(msecs)03d [%(levelname)s] " "%(name)s | %(funcName)s:%(lineno)d | %(message)s"),
        datefmt="%y-%m-%d %H:%M:%S",
        stream=sys.stdout,
    )

    tm = TMClient('test')

    cmd = {'cmd': Command.GET_USER, 'id': 2}
    response = tm.sendRequest(cmd)
    print(f"TMAdmin user received: {response}")

    cmd = {'cmd': Command.GET_TEAM, 'id': 20}
    response = tm.sendRequest(cmd)
    print(f"TMAdmin team received: {response}")

    cmd = {'cmd': Command.GET_ORG, 'id': 10}
    response = tm.sendRequest(cmd)
    print(f"TMAdmin organization received: {response}")

options: show_source: false heading_level: 3

Last update: March 6, 2024

API Docs for TM-Admin¶

tmadmin_manage.py¶

dbsupport.py¶

DBSupport ¶

connect async ¶

createTable async ¶

updateTable async ¶

resetSequence async ¶

getByID async ¶

getByName async ¶

getAll async ¶

getByWhere async ¶

getByLocation async ¶

deleteByID async ¶

getColumn async ¶

updateColumn async ¶

removeColumn async ¶

appendColumn async ¶

renameTable async ¶

copyTable async ¶

main async ¶

pgsupport.py¶

DBSupport ¶

connect async ¶

createTable async ¶

updateTable async ¶

resetSequence async ¶

getByID async ¶

getByName async ¶

getAll async ¶

getByWhere async ¶

getByLocation async ¶

deleteByID async ¶

getColumn async ¶

updateColumn async ¶

removeColumn async ¶

appendColumn async ¶

renameTable async ¶

copyTable async ¶

main async ¶

generator.py¶

Generator ¶

readConfig ¶

createTypes ¶

createSQLEnums ¶

createProtoEnums ¶

createPyEnums ¶

createPyMessage ¶

createPyClass ¶

createProtoMessage ¶

createSQLTable ¶

main ¶

proto.py¶

ProtoBuf ¶

createEnumProto ¶

createTableProto ¶

main ¶

tmserver.py¶

TMServer ¶

getTarget ¶

main ¶

tmclient.py¶

TMClient ¶

sendUserProfile ¶

getTarget ¶

main ¶

connect `async` ¶

createTable `async` ¶

updateTable `async` ¶

resetSequence `async` ¶

getByID `async` ¶

getByName `async` ¶

getAll `async` ¶

getByWhere `async` ¶

getByLocation `async` ¶

deleteByID `async` ¶

getColumn `async` ¶

updateColumn `async` ¶

removeColumn `async` ¶

appendColumn `async` ¶

renameTable `async` ¶

copyTable `async` ¶

main `async` ¶

connect `async` ¶

createTable `async` ¶

updateTable `async` ¶

resetSequence `async` ¶

getByID `async` ¶

getByName `async` ¶

getAll `async` ¶

getByWhere `async` ¶

getByLocation `async` ¶

deleteByID `async` ¶

getColumn `async` ¶

updateColumn `async` ¶

removeColumn `async` ¶

appendColumn `async` ¶

renameTable `async` ¶

copyTable `async` ¶

main `async` ¶