Make it a little faster, queue is in sqlite now, fix db bug and more

Tesses.YouTubeDownloader/src/YouTubeDownloader.tcross

@@ -67,6 +67,50 @@ class TYTD.Downloader {
 		this.Storage = vfs;
 		this.DatabaseDirectory = dbDir;
 	}
+
+	private PopQueue()
+	{
+		Mutex.Lock();
+		const db = OpenDB();
+		var res = Sqlite.Exec(db,"SELECT * FROM queue ORDER BY id DESC LIMIT 1;");
+		
+		if(TypeIsList(res) && res.Length > 0)
+		{
+			Sqlite.Exec(db,$"DELETE FROM queue WHERE id = {Sqlite.Escape(res[0].id)};");
+			res = res[0];
+		}
+		else { res = null;}
+		Sqlite.Close(db);
+		Mutex.Unlock();
+		
+		
+
+		if(TypeIsDictionary(res))
+		{
+			switch(res.resolution)
+			{
+				case Resolution.NoDownload:
+				{
+					this.PutVideoInfoIfNotExists(res.videoId);
+
+					return null;
+				}
+	
+				case Resolution.MP3:
+					return new TYTD.TranscodeAudio(res.videoId,".mp3");
+				case Resolution.FLAC:
+					return new TYTD.TranscodeAudio(res.videoId,".flac");
+				case Resolution.AMV:
+					return new TYTD.TranscodeAMV(res.videoId);
+				default:
+					return new TYTD.SDVideoDownload(res.videoId);
+			
+			}
+		}
+		return null;
+	}
+
+
 	/^
 		Download a video
 		id: video id or url
@@ -77,6 +121,8 @@ class TYTD.Downloader {
 		const theVideoId = TYTD.GetVideoId(id);
 		if(TypeIsString(theVideoId))
 		{
+
+			if(!TypeIsDefined(res)) res = Resolution.LowVideo;
 			const ent = {
 				Id = theVideoId,
 				Resolution = res,
@@ -89,43 +135,20 @@ class TYTD.Downloader {
 				return;
 			}
 		}
-
-		this.LOG($"Adding video: {TYTD.GetVideoId(id)}, original val: {id}");
-		switch(res)
-		{
-			case Resolution.NoDownload:
-			{
-				var id = TYTD.GetVideoId(id);
-				if(id != null)
-					this.PutVideoInfoIfNotExists(id);
-			}
-				break;
-			case Resolution.LowVideo:
-				this.Queue.Push(new TYTD.SDVideoDownload(id));
-				break;
-			case Resolution.VideoOnly:
-				this.Queue.Push(new TYTD.VOVideoDownload(id));
-				break;
-			
-			case Resolution.AudioOnly:
-				this.Queue.Push(new TYTD.AOVideoDownload(id));
-				break;
-			case Resolution.MP4:
-				this.Queue.Push(new TYTD.TranscodeVideo(id,".mp4"));
-				break;
-			case Resolution.MKV:
-				this.Queue.Push(new TYTD.TranscodeVideo(id,".mkv"));
-				break;
-			case Resolution.MP3:
-				this.Queue.Push(new TYTD.TranscodeAudio(id,".mp3"));
-				break;
-			case Resolution.FLAC:
-				this.Queue.Push(new TYTD.TranscodeAudio(id,".flac"));
-				break;
-			case Resolution.DontConvert:
-				this.Queue.Push(new TYTD.NoConvertVideoDownload(id));
-				break;
+		else {
+			this.LOG($"Malformed video id {id}");
+			return;
 		}
+
+
+		this.LOG($"Adding video: https://www.youtube.com/watch?v={theVideoId}");
+		//Sqlite.Exec(db,"CREATE TABLE IF NOT EXISTS queue (id INTEGER PRIMARY KEY AUTOINCREMENT, videoId TEXT, resolution TEXT);");
+		Mutex.Lock();
+		const db = OpenDB();
+		Sqlite.Exec(db, $"INSERT INTO queue (videoId, resolution) VALUES ({Sqlite.Escape(theVideoId)},{Sqlite.Escape(res)});");
+		Sqlite.Close(db);
+		Mutex.Unlock();
+		
 	}
 	/^
 		Download a playlist
@@ -287,6 +310,18 @@ class TYTD.Downloader {
 		if(views < 1000000000000) return $"{views/1000000000}B views";
 		return $"{views/1000000000000}T views";
 	}
+
+	public GetQueueItems(offset, count)
+	{
+		this.Mutex.Lock();
+		var db = this.OpenDB();
+		const vals = Sqlite.Exec(db,$"SELECT * FROM queue ORDER BY id DESC LIMIT {count} OFFSET {offset*count};");
+		Sqlite.Close(db);
+		this.Mutex.Unlock();
+		if(TypeIsList(vals))
+		return vals;
+		return [];
+	}
 	
 	/^
 		Get videos
@@ -295,7 +330,7 @@ class TYTD.Downloader {
 	^/
 	public GetVideos(query, offset, count)
 	{
-		this.Muxex.Lock();
+		this.Mutex.Lock();
 		var db = this.OpenDB();
 		
 		var q = Sqlite.Escape($"%{query}%");
@@ -330,7 +365,7 @@ class TYTD.Downloader {
 	^/
 	public GetPlaylists(query, offset, count)
 	{
-		this.Muxex.Lock();
+		this.Mutex.Lock();
 		var db = this.OpenDB();
 		var q = Sqlite.Escape($"%{query}%");
 		var res = Sqlite.Exec(db, $"SELECT * FROM playlists v WHERE (v.title LIKE {q}) LIMIT {count} OFFSET {offset*count};");
@@ -356,7 +391,7 @@ class TYTD.Downloader {
 			channelTitle="N/A",
 			items = []
 		};
-		this.Muxex.Lock();
+		this.Mutex.Lock();
 		var db = this.OpenDB();
 		var channelTitle = "";
 		var title = "";
@@ -420,7 +455,7 @@ class TYTD.Downloader {
 			authorName = "N/A",
 			items = []
 		};
-		this.Muxex.Lock();
+		this.Mutex.Lock();
 		var db = this.OpenDB();
 		var authorName = "";
 		
@@ -464,7 +499,7 @@ class TYTD.Downloader {
 	^/
 	public GetChannels(query, offset, count)
 	{
-		this.Muxex.Lock();
+		this.Mutex.Lock();
 		var db = this.OpenDB();
 		var q = Sqlite.Escape($"%{query}%");
 		var res = Sqlite.Exec(db, $"SELECT * FROM channels v WHERE (v.title LIKE {q}) LIMIT {count} OFFSET {offset*count};");
@@ -524,7 +559,7 @@ class TYTD.Downloader {
 		var id = TYTD.GetVideoId(vid);
 		
 		if(id == null) return null;
-		this.Muxex.Lock();
+		this.Mutex.Lock();
 		var db = this.OpenDB();
 		var res = Sqlite.Exec(db, $"SELECT * FROM videos WHERE videoId = {Sqlite.Escape(id)};");
 		
@@ -560,7 +595,7 @@ class TYTD.Downloader {
 	{
 		var id = TYTD.GetPlaylistId(vid);
 		if(id == null) return null;
-		this.Muxex.Lock();
+		this.Mutex.Lock();
 		var db = this.OpenDB();
 		var res = Sqlite.Exec(db, $"SELECT * FROM playlists WHERE playlistId = {Sqlite.Escape(id)};");
 		
@@ -582,7 +617,7 @@ class TYTD.Downloader {
 	{
 		var id = TYTD.GetChannelId(vid);
 		if(id == null) return null;
-		this.Muxex.Lock();
+		this.Mutex.Lock();
 		var db = this.OpenDB();
 		var res = Sqlite.Exec(db, $"SELECT * FROM channels WHERE channelId = {Sqlite.Escape(id)};");
 		
@@ -599,7 +634,7 @@ class TYTD.Downloader {
 	/^ Get the list of personal list names^/
 	public GetPersonalLists()
 	{
-		this.Muxex.Lock();
+		this.Mutex.Lock();
 		var db = this.OpenDB();
 		var items = [];
 
@@ -621,7 +656,7 @@ class TYTD.Downloader {
 	/^ Set the description of a personal list ^/	
 	public SetPersonalListDescription(name,description)
 	{
-		this.Muxex.Lock();
+		this.Mutex.Lock();
 		var db = this.OpenDB();
 
 		Sqlite.Exec(db, $"UPDATE personal_lists SET description = {Sqlite.Escape(description)} WHERE name = {Sqlite.Escape(name)};");
@@ -631,7 +666,7 @@ class TYTD.Downloader {
 	/^ Get the description of a personal list ^/
 	public GetPersonalListDescription(name)
 	{
-		this.Muxex.Lock();
+		this.Mutex.Lock();
 		var db = this.OpenDB();
 
 		var res = Sqlite.Exec(db, $"SELECT * FROM personal_lists WHERE name = {Sqlite.Escape(name)};");
@@ -648,7 +683,7 @@ class TYTD.Downloader {
 	}
 	public GetPersonalListTempUrl(name)
 	{
-		this.Muxex.Lock();
+		this.Mutex.Lock();
 		var db = this.OpenDB();
 		var items = [];
 
@@ -672,7 +707,7 @@ class TYTD.Downloader {
 	/^ ^/
 	public GetPersonalListContents(name, offset, count)
 	{
-		this.Muxex.Lock();
+		this.Mutex.Lock();
 		var db = this.OpenDB();
 		var items = [];
 
@@ -700,6 +735,23 @@ class TYTD.Downloader {
 
 		return items;
 	}
+
+	public QueueRemoveItem(id)
+	{
+		this.Mutex.Lock();
+		const db = OpenDB();
+		Sqlite.Exec(db,$"DELETE FROM queue WHERE id = {Sqlite.Escape(id)};");
+		Sqlite.Close(db);
+		this.Mutex.Unlock();
+	}
+	public QueueClear()
+	{
+		this.Mutex.Lock();
+		const db = OpenDB();
+		Sqlite.Exec(db,$"DELETE FROM queue;");
+		Sqlite.Close(db);
+		this.Mutex.Unlock();
+	}
 	
 	public AddToPersonalList(name, id)
 	{
@@ -822,12 +874,27 @@ class TYTD.Downloader {
 
 	private DownloaderThreadHandle;
 
-	private Queue = new TYTD.Queue();
 	/^
 		Get Video Queue count
 	^/
 
-	public getVideoQueueCount() this.Queue.Count;
+	public getVideoQueueCount() {
+		Mutex.Lock();
+		const db = OpenDB();
+
+		const res = Sqlite.Exec(db, "SELECT COUNT(*) FROM queue;");
+
+		Sqlite.Close(db);
+
+		Mutex.Unlock();
+
+		const queueCount = res[0].["COUNT(*)"];
+		if(TypeIsString(queueCount))
+		{
+			return ParseLong(queueCount);
+		}
+		return 0;
+	}
 
 	private PlaylistThreadHandle;
 
@@ -851,7 +918,7 @@ class TYTD.Downloader {
 		this.Mutex.Lock();
 		var db = this.OpenDB();
 		var _res = Sqlite.Exec(db,$"SELECT * FROM playlists p INNER JOIN playlist_entries e ON p.id = e.playlistId WHERE p.playlistId = {Sqlite.Escape(id)} LIMIT 1;");
-		
+		Sqlite.Close(db);
 		this.Mutex.Unlock();
 		if(TypeOf(_res) == "List" && _res.Length > 0)
 		{
@@ -868,7 +935,7 @@ class TYTD.Downloader {
 		this.Mutex.Lock();
 		var db = this.OpenDB();
 		var _res = Sqlite.Exec(db,$"SELECT * FROM videos WHERE channelId = {Sqlite.Escape(id)} LIMIT 1;");
-		
+		Sqlite.Close(db);
 		this.Mutex.Unlock();
 		if(TypeOf(_res) == "List" && _res.Length > 0)
 		{
@@ -1090,7 +1157,12 @@ class TYTD.Downloader {
 										var res = this.GetVideo(videoId);
 										if(res != null)
 											this.Bell.Invoke(this,{
-												Video = res
+												Video = {
+													VideoId = res.videoId,
+													ChannelId = res.channelId,
+													Title = res.title,
+													Channel = res.author
+												}
 											});
 									}
 									if(downloadRes != Resolution.NoDownload)
@@ -1134,11 +1206,13 @@ class TYTD.Downloader {
 		while(this.Running)
 		{
 			try {
-				var res = this.Queue.Pop();
+				var res = PopQueue();
+
+				//Console.WriteLine(res);
+				
 			
 				if(TypeOf(res) != "Null")
 				{
-				
 					if(TypeIsDefined(res.TYTD = this)){
 						res.Progress = (progress)=>{
 							this.CurrentVideoProgress = progress;
@@ -1153,13 +1227,14 @@ class TYTD.Downloader {
 						this.VideoStarted.Invoke(this,{
 							Video = res.Video
 						});
-				
 						res.Start();
 
 						this.VideoEnded.Invoke(this,{
 							Video = res.Video
 						});
+
 					}
+					
 				}
 			} catch(ex) {
 				try{
@@ -1402,7 +1477,7 @@ class TYTD.Downloader {
 	^/
 	public PutVideoInfo(info)
 	{
-		this.Muxex.Lock();
+		this.Mutex.Lock();
 		var db = this.OpenDB();
 
 		var keywords = info.keywords;
@@ -1420,7 +1495,7 @@ class TYTD.Downloader {
 	
 	private InitDatabase()
 	{
-		this.Muxex.Lock();
+		this.Mutex.Lock();
 		var db = this.OpenDB();
 		Sqlite.Exec(db,"CREATE TABLE IF NOT EXISTS videos (id INTEGER PRIMARY KEY AUTOINCREMENT, videoId TEXT UNIQUE, title TEXT, lengthSeconds INTEGER, keywords TEXT, channelId TEXT, shortDescription TEXT, viewCount INTEGER, author TEXT, addDate INTEGER, tytdTag TEXT);");
 		Sqlite.Exec(db,"CREATE TABLE IF NOT EXISTS playlists (id INTEGER PRIMARY KEY AUTOINCREMENT, playlistId TEXT UNIQUE,channelId TEXT,channelTitle TEXT, title TEXT);");
@@ -1433,6 +1508,7 @@ class TYTD.Downloader {
 		Sqlite.Exec(db,"CREATE TABLE IF NOT EXISTS users (id INTEGER PRIMARY KEY AUTOINCREMENT, username TEXT UNIQUE, password_hash TEXT, password_salt TEXT, flags INTEGER);");
 		Sqlite.Exec(db,"CREATE TABLE IF NOT EXISTS sessions (id INTEGER PRIMARY KEY AUTOINCREMENT, accountId INTEGER, key TEXT UNIQUE, expires INTEGER);");
 		Sqlite.Exec(db,"CREATE TABLE IF NOT EXISTS sso (id INTEGER PRIMARY KEY AUTOINCREMENT, service_name TEXT UNIQUE, service_pretty_name TEXT, sso_app_key TEXT UNIQUE, service_auth_post TEXT, service_auth_redirect TEXT);");
+		Sqlite.Exec(db,"CREATE TABLE IF NOT EXISTS queue (id INTEGER PRIMARY KEY AUTOINCREMENT, videoId TEXT, resolution TEXT);");
 		Sqlite.Exec(db,"ALTER TABLE sessions ADD expires INTEGER;");
 		Sqlite.Exec(db,"DELETE FROM sessions WHERE expires IS NULL;");
 		var config=Sqlite.Exec(db,"SELECT * FROM plugin_settings WHERE extension = '' AND key = 'settings';");
@@ -1455,7 +1531,7 @@ class TYTD.Downloader {
 	}
 	private _setPluginValue(extension,key,value)
 	{	
-		this.Muxex.Lock();
+		this.Mutex.Lock();
 		var db = this.OpenDB();
 		Sqlite.Exec(db, $"INSERT OR REPLACE INTO plugin_settings (extension,key,value) VALUES ({Sqlite.Escape(extension)},{Sqlite.Escape(key)},{Sqlite.Escape(value)});");
 		Sqlite.Close(db);
@@ -1701,7 +1777,6 @@ class TYTD.Downloader {
                         ],
                         Body = Net.Http.TextHttpRequestBody(embed("request2.json").ToString().Replace("VIDEO_ID_HERE", id).Replace("VISITOR_DATA",visitor),"application/json")
                     };
-					this.RateLimit();
                     var response = Net.Http.MakeRequest(url,requestData);
 				
                 if(response.StatusCode < 200 || response.StatusCode > 299) {
@@ -1734,13 +1809,14 @@ class TYTD.Downloader {
 
 				if(!TypeIsDictionary(jsonResp.playerResponse.videoDetails))
 				{
+					Console.WriteLine("YEY");
 					throw new VideoDownloadError(id, "videoDetails is missing");
 				}
 
-				if(!TypeIsList(jsonResp.playerResponse.streamingData.adaptiveFormats))
-				{
-					throw new VideoDownloadError(id, "adaptiveFormats is missing");
-				}
+				//if(!TypeIsList(jsonResp.playerResponse.streamingData.adaptiveFormats))
+				//{
+				//	throw new VideoDownloadError(id, "adaptiveFormats is missing");
+				//}
 				this.DownloadCaptions(jsonResp);
                 return jsonResp;
 		}
@@ -1835,7 +1911,7 @@ class TYTD.Downloader {
 
 		var first = true;
 
-		/*this.Muxex.Lock();
+		/*this.Mutex.Lock();
 		var db = this.OpenDB();
 		var d = $"INSERT INTO videos (videoId,title,lengthSeconds,keywords,channelId,shortDescription,viewCount,author,addDate,tytdTag) VALUES ({Sqlite.Escape(info.videoId)},{Sqlite.Escape(info.title)},{info.lengthSeconds},{Sqlite.Escape(info.keywords.ToString())},{Sqlite.Escape(info.channelId)},{Sqlite.Escape(info.shortDescription)},{info.viewCount},{Sqlite.Escape(info.author)},{DateTime.NowEpoch},{Sqlite.Escape(this.TYTDTag)});";
 		Sqlite.Exec(db, d);
@@ -1862,7 +1938,7 @@ class TYTD.Downloader {
 				};
 			*/
 			if(first) {
-				this.Muxex.Lock();
+				this.Mutex.Lock();
 				var db = this.OpenDB();
 				if(isPlaylist)
 				{
@@ -1898,7 +1974,7 @@ class TYTD.Downloader {
 				ids.Add(vid);
 				if(TypeOf(dbRow) == "Long")
 				{
-					this.Muxex.Lock();
+					this.Mutex.Lock();
 					var db = this.OpenDB();
 					Sqlite.Exec(db,$"INSERT INTO playlist_entries (playlistId,videoId) VALUES ({dbRow},{Sqlite.Escape(vid)});");
 					Sqlite.Close(db);
@@ -1919,21 +1995,21 @@ class TYTD.Downloader {
 
 	private requests = 0;
 
-	private rlm=new Muxex();
+	private rlm=new Mutex();
 
 	private RateLimit()
 	{
 		this.rlm.Lock();
 		var curRequest = DateTime.NowEpoch;
-		if((curRequest - this.lastRequest) > 60)
+		if((curRequest - this.lastRequest) > 10)
 		{
 			this.requests = 0;
 		}
 
 		this.requests++;
-		if(this.requests > 5)
+		if(this.requests >= 1)
 		{
-			DateTime.Sleep(25000);
+			DateTime.Sleep(1500);
 			this.requests=0;
 			curRequest = DateTime.NowEpoch;
 		}