From 78c2d0b294acb9492b37fc2b69a95b1f21e3d93b Mon Sep 17 00:00:00 2001 From: Danny McGee Date: Thu, 7 Mar 2024 21:33:16 -0500 Subject: [PATCH] `session`, `media_files/list` endpoints --- studio-frontend/src/app/app.element.ts | 34 ++- studio-web/src/lib/storyteller-api.ts | 293 ++++++++++++++++++++++++- utility/src/index.ts | 1 + utility/src/lib/assert.ts | 8 + 4 files changed, 327 insertions(+), 9 deletions(-) create mode 100644 utility/src/lib/assert.ts diff --git a/studio-frontend/src/app/app.element.ts b/studio-frontend/src/app/app.element.ts index 0c6627a..7f4a105 100644 --- a/studio-frontend/src/app/app.element.ts +++ b/studio-frontend/src/app/app.element.ts @@ -8,11 +8,12 @@ import { } from "@storyteller/studio"; import { ButtonElement } from "@storyteller/studio-ui/button"; import { + MediaFileType, SceneHierarchyElement, StorytellerApi, StudioURLParams, } from "@storyteller/studio-web"; -import { match } from "@storyteller/utility"; +import { assert, match } from "@storyteller/utility"; import { LitElement, html, nothing } from "lit"; import { customElement, state } from "lit/decorators.js"; import { classMap } from "lit/directives/class-map.js"; @@ -66,6 +67,37 @@ export class AppElement extends LitElement { return this; } + // TODO + override async connectedCallback(): Promise { + try { + const session = await this.api.session(); + console.log("session:", session); + + if (!session.logged_in) + return; // TODO: Prompt for login? + + assert(session.user?.username != null); + + const mediaFilesResult = await this.api.listMediaFiles({ + username: session.user.username, + filterMediaType: [ + MediaFileType.Bvh, + MediaFileType.Fbx, + MediaFileType.Glb, + MediaFileType.Gltf, + MediaFileType.Mocap, + MediaFileType.ScnRon, + ], + }); + console.log(mediaFilesResult); + } + catch (err) { + console.error(err); + } + + super.connectedCallback(); + } + @on("scene-saved") async onSceneSaved({ detail: scene }: SceneSavedEvent): Promise { try { diff --git a/studio-web/src/lib/storyteller-api.ts b/studio-web/src/lib/storyteller-api.ts index 714249d..5ec1cde 100644 --- a/studio-web/src/lib/storyteller-api.ts +++ b/studio-web/src/lib/storyteller-api.ts @@ -1,6 +1,4 @@ -import axios from "axios"; - -export type MediaToken = string; +import axios, { AxiosResponse } from "axios"; export class StorytellerApi { #api = axios.create({ @@ -8,10 +6,10 @@ export class StorytellerApi { withCredentials: true, }); - async createScene(file: File): Promise { + async createScene(file: File): Promise { type Response = { success: boolean; - media_file_token: MediaToken; + media_file_token: MediaFileToken; } const response = await this.#api.postForm("v1/engine/create_scene", { @@ -20,10 +18,289 @@ export class StorytellerApi { }, { headers: { Accept: "text/plain" }, }); + this.#validateResponse(response); - if (response.status === 200 && response.data.success) - return response.data.media_file_token; + return response.data.media_file_token; + } - throw new Error(`API responded with status ${response.status}: "${response.statusText}"`); + async session(): Promise { + const response = await this.#api.get("v1/session"); + this.#validateResponse(response); + + return response.data; + } + + async listMediaFiles(options: ListMediaFilesOptions = {}): Promise { + const params = {} as any; + + if (options.filterMediaType) + params.filter_media_type = options.filterMediaType.join(","); + if (options.sortAscending != null) + params.soer_ascending = options.sortAscending; + if (options.pageSize != null) + params.page_size = options.pageSize; + if (options.pageIndex != null) + params.page_index = options.pageIndex; + + const endpoint = options.username + ? `v1/media_files/list/user/${options.username}` + : "v1/media_files/list"; + + const response = await this.#api.get(endpoint, { params }); + this.#validateResponse(response); + + return response.data; + } + + #validateResponse(response: R): asserts response is OkResponse { + if (response.status !== 200) + throw new Error(`API responded with status ${response.status}: "${response.statusText}"`); + + if ( + response.data + && "success" in response.data + && response.data.success === false + ) { + throw new ApiError(response); + } } } + +export class ApiError extends Error { + detail: T; + + constructor (detail: T) { + super("API Error"); + this.detail = detail; + } +} + +type OkResponse = T & { status: 200 } + +export interface ListMediaFilesOptions { + username?: string; + filterMediaType?: MediaFileType[]; + pageSize?: number; + pageIndex?: number; + sortAscending?: boolean; +} + +export interface ListMediaFilesResult { + pagination: PaginationPage; + results: MediaFileForUserListItem[]; + success: boolean; +} + +/** + * Pagination by page id. + * + * This type of pagination is by "page id" and "page count". This should never + * be used to walk the entire public database as competitors and investors can + * use it to glean information about the scale of our service. This is best used + * for user profiles and scoped down results. + */ +export interface PaginationPage { + current: number; + total_page_count: number; +} + +export interface MediaFileListItem { + /** Date-time string */ + created_at: string; + creator_set_visibility: Visibility; + maybe_creator?: UserDetailsLight; + /** @deprecated */ + maybe_origin_model_token?: string; + maybe_origin_model_type?: MediaFileOriginModelType; + /** Text transcripts for TTS, etc. */ + maybe_text_transcript?: string; + media_type: MediaFileType; + origin: MediaFileOriginDetails; + origin_category: MediaFileOriginCategory; + origin_product_category: MediaFileOriginProductCategory; + /** URL to the media file. */ + public_bucket_path: string; + /** The primary key for Media Files */ + token: MediaFileToken; + /** Date-time string */ + updated_at: string; +} + +export type MediaFileForUserListItem = Omit; + +export enum Visibility { + Public = "public", + Hidden = "hidden", + Private = "private", +} + +/** + * Used in the `media_files` table in a `VARCHAR` field. + * + * DO NOT CHANGE VALUES WITHOUT A MIGRATION STRATEGY. + */ +export enum MediaFileOriginModelType { + RvcV2 = "rvc_v2", + SadTalker = "sad_talker", + SoVitsSvc = "so_vits_svc", + Tacotron2 = "tacotron2", + VallEX = "vall_e_x", + Rerender = "rerender", + MocapNet = "mocap_net", + ComfyUi = "comfy_ui", + StyleTts2 = "styletts2", + StableDiffusion_1_5 = "stable_diffusion_1_5", +} + +/** + * Used in the `media_files` table in a `VARCHAR(16)` field. + * + * DO NOT CHANGE VALUES WITHOUT A MIGRATION STRATEGY. + */ +export enum MediaFileType { + Audio = "audio", + Image = "image", + Video = "video", + Mocap = "mocap", + Bvh = "bvh", + Fbx = "fbx", + Glb = "glb", + Gltf = "gltf", + ScnRon = "scene_ron", +} + +/** Fields useful for enriching media file listings */ +export interface MediaFileOriginDetails { + maybe_model?: Model; + origin_category: MediaFileOriginCategory; + product_category: MediaFileOriginProductCategory; +} + +export interface Model { + model_type: MediaFileOriginModelType; + /** + * The model title (typically only populated for model_weights models, not + * legacy tables such as tts_models.) + */ + title?: string; + /** The primary key for the "model_weights" table. */ + token?: string; +} + +/** + * Used in the `media_files` table in a `VARCHAR` field. + * + * DO NOT CHANGE VALUES WITHOUT A MIGRATION STRATEGY. + */ +export enum MediaFileOriginCategory { + Inference = "inference", + Processed = "processed", + Upload = "upload", + DeviceApi = "device_api", + StoryEngine = "story_engine", +} + +/** + * Used in the `media_files` table in `VARCHAR(16)` field + * `origin_product_category`. + * + * This value indicates what product originally created the media file. (Not the + * ML model or user upload process.) This will let us scope media files to the + * product that generated them and filter them out of unrelated products if + * necessary (eg. a user probably doesn't want "Voice Designer" dataset samples + * in a video generation flow.) + * + * DO NOT CHANGE VALUES WITHOUT A MIGRATION STRATEGY. + */ +export enum MediaFileOriginProductCategory { + Unknown = "unknown", + FaceAnimator = "face_animator", + Tts = "tts", + VoiceConversion = "voice_conversion", + ZsVoice = "zs_voice", + VideoFilter = "video_filter", + Mocap = "mocap", + ImageGen = "image_gen", + Workflow = "workflow", +} + +export type MediaFileToken = string; + +export interface Session { + logged_in: boolean; + success: boolean; + user?: { + can_access_studio: boolean; + can_approve_w21_templates: boolean; + can_ban_users: boolean; + can_delete_other_users_tts_models: boolean; + can_delete_other_users_tts_results: boolean; + can_delete_other_users_w2l_results: boolean; + can_delete_other_users_w2l_templates: boolean; + can_delete_own_account: boolean; + can_delete_own_tts_models: boolean; + can_delete_own_tts_results: boolean; + can_delete_own_w2l_results: boolean; + can_delete_own_w2l_templates: boolean; + can_delete_users: boolean; + can_edit_other_users_profiles: boolean; + can_edit_other_users_tts_models: boolean; + can_edit_other_users_w2l_templates: boolean; + can_upload_tts_models: boolean; + can_upload_w2l_templates: boolean; + can_use_tts: boolean; + can_use_w2l: boolean; + core_info: UserDetailsLight; + display_name: string; + email_gravatar_hash: string; + fakeyou_plan: FakeYouPlan; + maybe_feature_flags: unknown; + storyteller_stream_plan: StorytellerStreamPlan; + user_token: string; + username: string; + }; +} + +/** + * Everything we need to refer to a user on the public web interface. + */ +export interface UserDetailsLight { + default_avatar: DefaultAvatarInfo; + /** + * As of 2023-08-23, this is the username with capitalization (In the future, + * a display name can be customized by the user.) + */ + display_name: string; + /** + * Email hash for Gravatar Always set for now since login is email/username + + * password. In the future this will need to become an optional OR be filled + * with a bogus hash. + */ + gravatar_hash: string; + /** The primary key for users. */ + user_token: string; + /** + * The unique username someone logs in with. As of 2023-08-23, this is always + * lowercase. + */ + username: string; +} + +export interface DefaultAvatarInfo { + color_index: number; + image_index: number; +} + +export enum FakeYouPlan { + Free = "free", + Basic = "basic", + Standard = "standard", + Pro = "pro", +} + +export enum StorytellerStreamPlan { + Free = "free", + Basic = "basic", + Standard = "standard", + Pro = "pro", +} diff --git a/utility/src/index.ts b/utility/src/index.ts index 1c552cf..e35da3a 100644 --- a/utility/src/index.ts +++ b/utility/src/index.ts @@ -1,3 +1,4 @@ +export * from "./lib/assert"; export * from "./lib/exists"; export * from "./lib/match"; export * from "./lib/types"; diff --git a/utility/src/lib/assert.ts b/utility/src/lib/assert.ts new file mode 100644 index 0000000..1da67e9 --- /dev/null +++ b/utility/src/lib/assert.ts @@ -0,0 +1,8 @@ +export function assert(expression: unknown, message?: string): asserts expression { + if (!expression) { + let err = "Assertion failed"; + if (message) err += `: ${message}`; + + throw new Error(err); + } +}