Merge pull request #425 from classtranscribe/staging

PushToProduction

ClassTranscribeDatabase/CTDbContext.cs

@@ -74,12 +74,13 @@ public static string ConnectionStringBuilder()
  // TODO: Max MaxPoolSize and port should be configurable
 
  var configurations = CTDbContext.GetConfigurations();
- return "Server=" + configurations["POSTGRES_SERVER_NAME"]
- + ";Port=5432"
- + ";Database=" + configurations["POSTGRES_DB"]
- + ";User Id=" + configurations["ADMIN_USER_ID"]
- + ";Password=" + configurations["ADMIN_PASSWORD"]
- + ";MaxPoolSize=1000;";
+ string conn = $"Server={configurations["POSTGRES_SERVER_NAME"]};"
+ + $"Port={configurations["POSTGRES_SERVER_PORT"] ?? "5432"};"
+ + $"Database={configurations["POSTGRES_DB"]};"
+ + $"User Id={configurations["ADMIN_USER_ID"]};"
+ + $"Password={configurations["ADMIN_PASSWORD"]};"
+ + $"MaxPoolSize={configurations["POSTGRES_CLIENT_MAX_POOL_SIZE"] ?? "1000"};";
+ return conn;
  }
 
  /// <summary>
@@ -112,21 +113,67 @@ public static CTDbContext CreateDbContext()
  /// <returns> The configurations </returns>
  public static IConfiguration GetConfigurations()
  {
+ var basedir = System.IO.Directory.GetCurrentDirectory();
+
+ if (String.IsNullOrEmpty(Environment.GetEnvironmentVariable("POSTGRES_DB")))
+ {
+ LoadEnvFileIfExists($"{basedir}/../../../LocalEnvironmentVariables.txt");
+ }
+
  var configuration = new ConfigurationBuilder().AddEnvironmentVariables().Build();
 
  if (configuration.GetValue<string>("DEV_ENV", "NULL") != "DOCKER")
  {
- string path = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
  string appSettingsFileName = "vs_appsettings.json";
+ string path = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
 
  if (File.Exists(path + Path.DirectorySeparatorChar + appSettingsFileName))
  {
  return new ConfigurationBuilder().SetBasePath(path).AddJsonFile(appSettingsFileName).Build();
- }
+ } 
+
  }
 
  return configuration;
  }
+ public static string DropQuotes(string s)
+ {
+ char first = s.Length < 2 ? 'a': s[0];
+ if (! "\"'".Contains(first))
+ {
+ return s;
+ }
+ char last = s[^1];
+ if(first == last)
+ {
+ return s[1..^1].Replace($"\\{first}", first.ToString());
+ }
+ return s;
+ }
+ public static bool LoadEnvFileIfExists(string filePath)
+ {
+ if (!File.Exists(filePath)) {
+ Console.WriteLine($"Env file {filePath} not found - ignoring");
+ return false;
+ }
+ var count = 0;
+ foreach (var line in File.ReadAllLines(filePath))
+ {
+ if (!line.Contains("=") || line.TrimStart().StartsWith("#"))
+ continue;
+
+ var parts = line.Split( '=', 2);
+ var key = parts[0].Trim();
+ var val = DropQuotes( parts[1].Trim());
+ //Console.WriteLine($"{key}:{val.Length} chars");
+
+ Environment.SetEnvironmentVariable(key,val);
+ count += 1;
+ }
+ Console.WriteLine($"{count} environment variables set using {filePath}");
+ return true;
+ }
+
  protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
  {
  optionsBuilder.UseNpgsql(ConnectionStringBuilder());
@@ -268,7 +315,7 @@ public override int SaveChanges(bool acceptAllChangesOnSuccess)
  return base.SaveChanges(acceptAllChangesOnSuccess);
  }
 
- public override Task<int> SaveChangesAsync(bool acceptAllChangesOnSuccess, CancellationToken cancellationToken = default(CancellationToken))
+ public override Task<int> SaveChangesAsync(bool acceptAllChangesOnSuccess, CancellationToken cancellationToken = default)
  {
  OnBeforeSaving();
  return base.SaveChangesAsync(acceptAllChangesOnSuccess, cancellationToken);

ClassTranscribeDatabase/CaptionQueries.cs

@@ -25,7 +25,8 @@ public async Task<List<Caption>> GetCaptionsAsync(string videoId, string languag
  {
  try
  {
- var transcriptionId = _context.Transcriptions.Where(t => t.Language == language && t.VideoId == videoId).First().Id;
+ var transcriptionId = _context.Transcriptions.Where(t => t.Language == language && t.VideoId == videoId
+ && t.TranscriptionType == TranscriptionType.Caption).First().Id;
  return await GetCaptionsAsync(transcriptionId);
  }
  catch (System.InvalidOperationException)

ClassTranscribeDatabase/CommonUtils.cs

@@ -34,7 +34,11 @@ public enum TaskType
  BuildElasticIndex = 16,
  ExampleTask = 17,
  CleanUpElasticIndex = 18,
- PythonCrawler = 19 
+ PythonCrawler = 19,
+
+ DescribeVideo = 20,
+ DescribeImage = 21
+
  }
 
  public class Languages

ClassTranscribeDatabase/Globals.cs

@@ -13,6 +13,7 @@ public class AppSettings
  public string JWT_KEY { get; set; }
  public string ALLOWED_HOSTS { get; set; }
  public string POSTGRES_SERVER_NAME { get; set; }
+ public string POSTGRES_SERVER_PORT { get; set; } = "5432";
  public string POSTGRES_DB { get; set; }
 
  public string ADMIN_USER_ID { get; set; }

ClassTranscribeDatabase/Models/Caption.cs

@@ -13,11 +13,18 @@ public enum CaptionType
  TextCaption = 0,
  AudioDescription = 1
  }
+
+ public static class CaptionConstants {
+ public const string PlaceHolderText = "...Processing...";
+
+ }
  /// <summary>
  /// Each line of caption is stored as a row in the database.
  /// </summary>
  public class Caption : Entity
  {
+
+ public bool HasPlaceHolderText() { return this.Text == CaptionConstants.PlaceHolderText; }
  public int Index { get; set; }
  public TimeSpan Begin { get; set; }
  public TimeSpan End { get; set; }
@@ -115,7 +122,7 @@ public static List<Caption> ToCaptionEntitiesInterpolate(int captionsCount, Time
  else
  {
  caption = tempCaption.Substring(0, index);
- tempCaption = tempCaption.Substring(index);
+ tempCaption = tempCaption[index..];
  tempCaption = tempCaption.Trim();
  }
  curEnd = curBegin.Add(new TimeSpan(0, 0, 0, 0, newDuration));
@@ -140,8 +147,8 @@ public static List<Caption> ToCaptionEntitiesInterpolate(int captionsCount, Time
  End = curEnd,
  Text = tempCaption
  });
- curBegin = curEnd;
- curDuration = End.Subtract(curBegin);
+ // curBegin = curEnd;
+ // curDuration = End.Subtract(curBegin);
  }
  return captions;
  }
@@ -204,7 +211,7 @@ public static string GenerateWebVTTString(List<Caption> captions, string languag
  // 
  public static string GenerateDXFPString(List<Caption> captions, string language)
  {
- string now = DateTime.UtcNow.ToString("o", System.Globalization.CultureInfo.InvariantCulture);
+ // string now = DateTime.UtcNow.ToString("o", System.Globalization.CultureInfo.InvariantCulture);
  string header = @"
 <?xml version=""1.0"" encoding=""utf-8""?>
 <tt xml:lang=""en"" xmlns=""http://www.w3.org/ns/ttml""

ClassTranscribeDatabase/Models/Models.cs

@@ -147,6 +147,8 @@ public ResourceType GetResourceType()
  case Playlist _: return ResourceType.Playlist;
  case Media _: return ResourceType.Media;
  case EPub _: return ResourceType.EPub;
+ default:
+ break;
  }
  throw new InvalidOperationException("Invalid Type passed" + this);
  }
@@ -243,12 +245,12 @@ public class Playlist : Entity
  public JObject JsonMetadata { get; set; } = new JObject(); // for serverside use
  public string Options { get; set; } = "";
 
- public virtual JObject getOptionsAsJson() {
+ public virtual JObject GetOptionsAsJson() {
  return String.IsNullOrEmpty(this.Options) ? new JObject() : JObject.Parse(this.Options);
  }
 
- public virtual void setOptionsAsJson(JObject json) {
- this.Options = json.ToString();
+ public virtual void SetOptionsAsJson(JObject json) {
+ this.Options = json.ToString(Newtonsoft.Json.Formatting.None);
  }
  public int Index { get; set; }
  public Visibility Visibility { get; set; }
@@ -281,11 +283,11 @@ public class Media : Entity
 
  public string Options { get; set; } = "";
 
- public virtual JObject getOptionsAsJson() {
+ public virtual JObject GetOptionsAsJson() {
  return String.IsNullOrEmpty(this.Options) ? new JObject() : JObject.Parse(this.Options);
  }
 
- public virtual void setOptionsAsJson(JObject json) {
+ public virtual void SetOptionsAsJson(JObject json) {
  this.Options = json.ToString(Newtonsoft.Json.Formatting.None);
  }
  }
@@ -558,22 +560,22 @@ public class TextData : Entity
  public string? Text {get; set;}
 
 
- public void setFromJSON(JToken? o) {
+ public void SetFromJSON(JToken? o) {
  if(o == null) {
  Text = null;
  return;
  }
  Text = o.ToString(Newtonsoft.Json.Formatting.None);
  }
 
- public JToken? getAsJSON() { 
+ public JToken? GetAsJSON() { 
  if( string.IsNullOrEmpty(Text)) {
  return null;
  }
  return JToken.Parse(Text);
  }
 
- public JArray? getAsJArray()
+ public JArray? GetAsJArray()
  {
  if (string.IsNullOrEmpty(Text))
  {

ClassTranscribeDatabase/Services/MSTranscription/KeyProvider.cs

@@ -23,14 +23,14 @@ public class Key
  /// </summary>
  public class KeyProvider
  {
- private AppSettings _appSettings;
- private List<Key> Keys;
- private HashSet<string> CurrentVideoIds;
+ private readonly AppSettings _appSettings;
+ private readonly List<Key> Keys;
+ private readonly HashSet<string> CurrentVideoIds;
 
  public KeyProvider(AppSettings appSettings)
  {
  _appSettings = appSettings;
- string subscriptionKeys = _appSettings.AZURE_SUBSCRIPTION_KEYS;
+ string subscriptionKeys = _appSettings.AZURE_SUBSCRIPTION_KEYS ?? "";
  Keys = new List<Key>();
  CurrentVideoIds = new HashSet<string>();