Fixed / Refactored SysZip to actually work, and store compressed entries without uncompressing an entire zip (#761)

* well boys I fixed it. you can now have Zipped Mods!!!!!

* comment update

* quick fixes + optimizations

* fixed stream audio bug????

* found the issue ffs

* removing traces oops

* fixed videos from not being able to play. Inspired from VideoCutscene's fix for ZIP videos, and also removed that fix in favor for just fixing the issue entirely

* oop just in case no special characters that kill themself on other shit

* Implemented preloading every video (caching) when loading a ZipFolderLibrary.

Since videos have to be decompressed and be saved as a file anyways this reduces the time to do that, and what not.

* [UNTESTED] Allowed Zip Extensions for allowing any extension in the list to be passed as a `.zip`

* After some more testing loading Zip Addons works as well.

* Added the long awaited .cnemod file type!!!!! (it's just a folder with .cnemod with a .zip inside LMAO)

* Chalking up the Streamed Audio Bug to a mod issue, and also added capabilites to detect if a library is considered compressed

* Added warning to those who try to use the editor with compressed libraries

* ok instead of making it a .cnemod, the zip name will be `cnemod.zip` or what not.

* removing unnessesary traces oops

* clean Library accessor ig then

* fixed small issues and removed an idiot idea

* writing todos and fixing some context that left the room in comments

* Fixed an issue where if you had a file of 0 bytes (nothing) in a zipped library it would FUCKING CRASH the game

* Actually fixed the issue of decompressing 0 byte files

Also stopped saving the data for Folders in Zipped files, as they are 0 bytes anyways.

* Added the features that needed to exist, and removed a stupid line I saw in `ModsFolder` lol

For some reason animation offsets for characters are breaking and I can't tell if its my drive killing itself or cne so uh

* fix an issue that will force the game to crash if first ever mod being loaded is a ZipModLibrary

Author: ItsLJcool

source/funkin/backend/assets/AssetsLibraryList.hx

@@ -8,7 +8,19 @@ import lime.utils.AssetLibrary;
 import haxe.ds.Map;
 
 class AssetsLibraryList extends AssetLibrary {
+
 	public var libraries:Array<AssetLibrary> = [];
+	public var cleanLibraries(get, never):Array<AssetLibrary>;
+	function get_cleanLibraries():Array<AssetLibrary> {
+		return [for (l in libraries) getCleanLibrary(l)];
+	}
+	
+	// is true if any library in `libraries` contains some kind of compressed library. 
+	public var hasCompressedLibrary(get, never):Bool;
+	function get_hasCompressedLibrary():Bool {
+		for (l in libraries) if (getCleanLibrary(l).isCompressed) return true;
+		return false;
+	}
 
 	@:allow(funkin.backend.system.Main)
 	@:allow(funkin.backend.system.MainState)

source/funkin/backend/assets/ModsFolder.hx

@@ -83,10 +83,11 @@ class ModsFolder {
 	 */
 	public static function loadModLib(path:String, force:Bool = false, ?modName:String) {
 		#if MOD_SUPPORT
-		if (FileSystem.exists('$path.zip'))
-			return loadLibraryFromZip('$path'.toLowerCase(), '$path.zip', force, modName);
-		else
-			return loadLibraryFromFolder('$path'.toLowerCase(), '$path', force, modName);
+		for (ext in Flags.ALLOWED_ZIP_EXTENSIONS) {
+			if (!FileSystem.exists('$path.$ext')) continue;
+			return loadLibraryFromZip('$path'.toLowerCase(), '$path.$ext', force, modName);
+		}
+		return loadLibraryFromFolder('$path'.toLowerCase(), '$path', force, modName);
 
 		#else
 		return null;
@@ -96,27 +97,16 @@ class ModsFolder {
 	public static function getModsList():Array<String> {
 		var mods:Array<String> = [];
 		#if MOD_SUPPORT
-		if (!FileSystem.exists(modsPath)) {
-			// Mods directory does not exist yet, create it
-			FileSystem.createDirectory(modsPath);
-		}
+		// Mods directory does not exist yet, create it
+		if (!FileSystem.exists(modsPath)) FileSystem.createDirectory(modsPath);
 
 		final modsList:Array<String> = FileSystem.readDirectory(modsPath);
 
-		if (modsList == null || modsList.length <= 0)
-			return mods;
+		if (modsList == null || modsList.length <= 0) return mods;
 
 		for (modFolder in modsList) {
-			if (FileSystem.isDirectory(modsPath + modFolder)) {
-				mods.push(modFolder);
-			} else {
-				var ext = Path.extension(modFolder).toLowerCase();
-				switch(ext) {
-					case 'zip':
-						// is a zip mod!!
-						mods.push(Path.withoutExtension(modFolder));
-				}
-			}
+			if (FileSystem.isDirectory(modsPath + modFolder)) mods.push(modFolder);
+			else if (Flags.ALLOWED_ZIP_EXTENSIONS.contains(Path.extension(modFolder))) mods.push(Path.withoutExtension(modFolder));
 		}
 		#end
 		return mods;
@@ -128,7 +118,9 @@ class ModsFolder {
 			#if TRANSLATIONS_SUPPORT
 			if(skipTranslated && (l is TranslatedAssetLibrary)) continue;
 			#end
-			if (l is ScriptedAssetLibrary || l is IModsAssetLibrary) libs.push(cast(l, IModsAssetLibrary));
+			// No need to check for it being a `ScriptedAssetLibrary`, if `ScriptedAssetLibrary` extends ModsFolderLibrary, which implements `IModsAssetLibrary`
+			// If you have to revert this change then uhhhhh wasn't me, trust 🙏
+			if (/*l is ScriptedAssetLibrary ||*/ l is IModsAssetLibrary) libs.push(cast(l, IModsAssetLibrary));
 		}
 		return libs;
 	}

source/funkin/backend/assets/ZipFolderLibrary.hx

@@ -1,11 +1,14 @@
 package funkin.backend.assets;
 
+import funkin.backend.system.Flags;
+
 import haxe.io.Path;
 import lime.graphics.Image;
 import lime.media.AudioBuffer;
 import lime.text.Font;
 import lime.utils.Bytes;
 import openfl.utils.AssetLibrary;
+import sys.io.File;
 
 #if MOD_SUPPORT
 import funkin.backend.utils.SysZip.SysZipEntry;
@@ -15,40 +18,76 @@ class ZipFolderLibrary extends AssetLibrary implements IModsAssetLibrary {
 	public var basePath:String;
 	public var modName:String;
 	public var libName:String;
-	public var useImageCache:Bool = false;
 	public var prefix = 'assets/';
-
+	
 	public var zip:SysZip;
 	public var assets:Map<String, SysZipEntry> = [];
 	public var lowerCaseAssets:Map<String, SysZipEntry> = [];
 	public var nameMap:Map<String, String> = [];
 
-	public function new(basePath:String, libName:String, ?modName:String) {
+	public var PRELOAD_VIDEOS:Bool = true;
+
+	public function new(basePath:String, libName:String, ?modName:String, ?preloadVideos:Bool = true) {
 		this.libName = libName;
 
 		this.basePath = basePath;
 
 		this.modName = (modName == null) ? libName : modName;
 
 		zip = SysZip.openFromFile(basePath);
-		zip.read();
 		for(entry in zip.entries) {
-			if (entry.fileName.length < 0 || entry.fileName.endsWith("/"))
-				continue;
+			if (entry.fileName.length < 0 || entry.fileName.endsWith("/")) continue;
 
-			lowerCaseAssets[entry.fileName.toLowerCase()] = assets[entry.fileName.toLowerCase()] = assets[entry.fileName] = entry;
-			nameMap.set(entry.fileName.toLowerCase(), entry.fileName);
+			var name:String = entry.fileName.toLowerCase(); // calling .toLowerCase a million times is never the solution
+			lowerCaseAssets[name] = assets[name] = assets[entry.fileName] = entry;
+			nameMap.set(name, entry.fileName);
 		}
 
 		super();
+
+		isCompressed = true;
+		
+		// don't override default value of true if the file exists.
+		// by default `PRELOAD_VIDEOS` is true so you will never need to add this file, but in the case of it being false this is a backup method.
+		PRELOAD_VIDEOS = (!PRELOAD_VIDEOS) ? exists("assets/data/PRECACHE_VIDEOS", "TEXT") : PRELOAD_VIDEOS;
+
+		// if (PRELOAD_VIDEOS) precacheVideos(); // we do this in `MainState` now to handle for `Flags.VIDEO_EXT` :)
+	}
+
+	public function precacheVideos() {
+		_videoExtensions = [Flags.VIDEO_EXT];
+		
+		videoCacheRemap = [];
+		for (entry in zip.entries) {
+			var name = entry.fileName.toLowerCase();
+			if (_videoExtensions.contains(Path.extension(name))) getPath(prefix+name);
+		}
+
+		var count:Int = 0;
+        for (_ in videoCacheRemap.keys()) count++;
+		if (count <= 0) return;
+		trace('Precached $count video${(count == 1) ? "" : "s"}');
+	}
+
+	// Now we have supports for videos in ZIP!!
+	public var _videoExtensions:Array<String> = [Flags.VIDEO_EXT];
+	public var videoCacheRemap:Map<String, String> = [];
+	public function getVideoRemap(originalPath:String):String {
+		if (!_videoExtensions.contains(Path.extension(_parsedAsset))) return originalPath;
+		if (videoCacheRemap.exists(originalPath)) return videoCacheRemap.get(originalPath);
+
+		// We adding the length of the string to counteract folder in folder naming duplicates.
+		var newPath = './.temp/${_parsedAsset.length}-zipvideo-${_parsedAsset.split("/").pop()}';
+		File.saveBytes(newPath, unzip(assets[_parsedAsset]));
+		videoCacheRemap.set(originalPath, newPath);
+		return newPath;
 	}
 
 	function toString():String {
-		return '(ZipFolderLibrary: $libName/$modName)';
+		return '(ZipFolderLibrary: $libName/$modName | ${zip.entries.length} entries | Detected Video Extensions: ${_videoExtensions.join(", ")})';
 	}
 
 	public var _parsedAsset:String;
-
 	public override function getAudioBuffer(id:String):AudioBuffer {
 		__parseAsset(id);
 		return AudioBuffer.fromBytes(unzip(assets[_parsedAsset]));
@@ -71,15 +110,12 @@ class ZipFolderLibrary extends AssetLibrary implements IModsAssetLibrary {
 		return getAssetPath();
 	}
 
-
-
-	public inline function unzip(f:SysZipEntry)
-		return f == null ? null : zip.unzipEntry(f);
+	public inline function unzip(f:SysZipEntry) return (f == null) ? null : zip.unzipEntry(f);
 
 	public function __parseAsset(asset:String):Bool {
 		if (!asset.startsWith(prefix)) return false;
 		_parsedAsset = asset.substr(prefix.length);
-		if(ModsFolder.useLibFile) {
+		if (ModsFolder.useLibFile) {
 			var file = new haxe.io.Path(_parsedAsset);
 			if(file.file.startsWith("LIB_")) {
 				var library = file.file.substr(4);
@@ -90,8 +126,7 @@ class ZipFolderLibrary extends AssetLibrary implements IModsAssetLibrary {
 		}
 
 		_parsedAsset = _parsedAsset.toLowerCase();
-		if(nameMap.exists(_parsedAsset))
-			_parsedAsset = nameMap.get(_parsedAsset);
+		if (nameMap.exists(_parsedAsset)) _parsedAsset = nameMap.get(_parsedAsset);
 		return true;
 	}
 
@@ -106,9 +141,8 @@ class ZipFolderLibrary extends AssetLibrary implements IModsAssetLibrary {
 		return assets[_parsedAsset] != null;
 	}
 
-	private function getAssetPath() {
-		trace('[ZIP]$basePath/$_parsedAsset');
-		return '[ZIP]$basePath/$_parsedAsset';
+	private inline function getAssetPath() {
+		return getVideoRemap('$basePath/$_parsedAsset');
 	}
 
 	// TODO: rewrite this to 1 function, like ModsFolderLibrary
@@ -157,18 +191,6 @@ class ZipFolderLibrary extends AssetLibrary implements IModsAssetLibrary {
 		return content;
 	}
 
-	public override function list(type:String):Array<String> {
-		return[for(k=>e in nameMap) '$prefix$e'];
-	}
-
-	// Backwards compat
-
-	@:noCompletion public var zipPath(get, set):String;
-	@:noCompletion private inline function get_zipPath():String {
-		return basePath;
-	}
-	@:noCompletion private inline function set_zipPath(value:String):String {
-		return basePath = value;
-	}
+	public override function list(type:String):Array<String> { return [for(k=>e in nameMap) '$prefix$e']; }
 }
 #end

source/funkin/backend/scripting/GlobalScript.hx

@@ -106,12 +106,15 @@ class GlobalScript {
 	public static function onModSwitch(newMod:String) {
 		destroy();
 		scripts = new ScriptPack("GlobalScript");
-		for (i in funkin.backend.assets.ModsFolder.getLoadedMods()) {
-			var path = Paths.script('data/global/LIB_$i');
+		for (lib in funkin.backend.assets.ModsFolder.getLoadedModsLibs()) {
+			var modName = lib.modName;
+			var path = Paths.script('data/global/LIB_$modName');
 			var script = Script.create(path);
-			if (script is DummyScript)
-				continue;
-			script.remappedNames.set(script.fileName, '$i:${script.fileName}');
+			if (script is DummyScript) continue;
+			script.remappedNames.set(script.fileName, '$modName:${script.fileName}');
+			// so you can get the current mod's library in GloablScript :)
+			// you should not make this a static variable then all scripts will try to reference the 1 static variable, which will be overwritten :yoikes:
+			script.set("MOD_LIBRARY", lib);
 			scripts.add(script);
 			script.load();
 		}

source/funkin/backend/system/Flags.hx

@@ -19,9 +19,11 @@ class Flags {
 
 	// -- Codename's Addon Config --
 	@:bypass public static var addonFlags:Map<String, Dynamic> = [];
-
 	public static var CURRENT_API_VERSION:Int = 2;
 
+	// -- Codename's ZipFolderLibrary Config --
+	public static var ALLOWED_ZIP_EXTENSIONS:Array<String> = ["zip"];
+
 	// -- Codename's Mod Config --
 	public static var MOD_NAME:String = "";
 	public static var MOD_DESCRIPTION:String = "";

source/funkin/backend/system/MainState.hx

@@ -7,12 +7,14 @@ import flixel.FlxState;
 import funkin.backend.assets.AssetsLibraryList;
 import funkin.backend.assets.ModsFolder;
 import funkin.backend.assets.ModsFolderLibrary;
+import funkin.backend.assets.ZipFolderLibrary;
 import funkin.backend.chart.EventsData;
 import funkin.backend.system.framerate.Framerate;
 import funkin.editors.ModConfigWarning;
 import funkin.menus.TitleState;
 import haxe.io.Path;
 
+
 @dox(hide)
 typedef AddonInfo = {
 	var name:String;
@@ -57,12 +59,32 @@ class MainState extends FlxState {
 		var _highPriorityAddons:Array<AddonInfo> = [];
 		var _noPriorityAddons:Array<AddonInfo> = [];
 
+		var quick_modsPath = ModsFolder.modsPath + ModsFolder.currentModFolder;
+
+		// handing if the loading mod (before it's properly loaded) is a compressed mod
+		// we just need to use `Paths.assetsTree.hasCompressedLibrary` to complete valid checks for actual loaded compressed mods
+		var isZipMod = false;
+		
+		// If we know it's a compressed mod, then we can check if it's using the `cnemod` folder path.
+		// All it is really is a folder with the mod's name, then a compressed file called "cnemod.[zip|7z|rar|etc]"
+		var isCneMod = false;
+
+		// We are doing it like this because think about it: it's 1 for loop lol
+		// We just need to know if any of these values is true, so if only one is true and we are not close to being done in the loop, that's fine.
+		// 
+		for (ext in Flags.ALLOWED_ZIP_EXTENSIONS) {
+			if (FileSystem.exists(quick_modsPath+"."+ext)) isZipMod = true;
+			if (FileSystem.exists(quick_modsPath+"/cnemod."+ext)) isCneMod = true;
+			if (isZipMod && isCneMod) break;
+		}
+		
+		// We get the addons folder from relative space (`./`) and then our mod's addons.
 		var addonPaths = [
 			ModsFolder.addonsPath,
-			(
-				ModsFolder.currentModFolder != null ?
-					ModsFolder.modsPath + ModsFolder.currentModFolder + "/addons/" :
-					null
+			// So to check the mod's addons folder, we need to decompress it. Which is impossible* in this stage of the loading library process.
+			// TODO: Write a function when the library is loaded to decompress the contents and then load the libraries :)
+			( (ModsFolder.currentModFolder != null && !isZipMod) ?
+				quick_modsPath + "/addons/" : null
 			)
 		];
 
@@ -72,12 +94,8 @@ class MainState extends FlxState {
 
 			for (addon in FileSystem.readDirectory(path)) {
 				if (!FileSystem.isDirectory(path + addon)) {
-					switch(Path.extension(addon).toLowerCase()) {
-						case 'zip':
-							addon = Path.withoutExtension(addon);
-						default:
-							continue;
-					}
+					if (Flags.ALLOWED_ZIP_EXTENSIONS.contains(Path.extension(addon))) addon = Path.withoutExtension(addon);
+					else continue;
 				}
 
 				var data:AddonInfo = {
@@ -100,9 +118,14 @@ class MainState extends FlxState {
 		#if MOD_SUPPORT
 		for (addon in _lowPriorityAddons)
 			loadLib(addon.path, ltrim(addon.name, "[LOW]"));
-
-		if (ModsFolder.currentModFolder != null)
-			loadLib(ModsFolder.modsPath + ModsFolder.currentModFolder, ModsFolder.currentModFolder);
+		
+		if (ModsFolder.currentModFolder != null) {
+			// isCneMod is a guarentee to be a zip mod because we just checked for it, so this will always load as a CompressedLibrary
+			if (isCneMod)
+				loadLib(quick_modsPath + "/cnemod", ModsFolder.currentModFolder);
+			else
+				loadLib(quick_modsPath, ModsFolder.currentModFolder);
+		}
 
 		for (addon in _noPriorityAddons)
 			loadLib(addon.path, addon.name);
@@ -134,9 +157,15 @@ class MainState extends FlxState {
 		CoolUtil.safeAddAttributes('./.temp/', NativeAPI.FileAttribute.HIDDEN);
 		#end
 
+		for (lib in ModsFolder.getLoadedModsLibs()) {
+			if (!(lib is ZipFolderLibrary)) continue;
+			if (cast(lib, ZipFolderLibrary).PRELOAD_VIDEOS) cast(lib, ZipFolderLibrary).precacheVideos();
+		}
+
 		var startState:Class<FlxState> = Flags.DISABLE_WARNING_SCREEN ? TitleState : funkin.menus.WarningState;
 
-		if (Options.devMode && Options.allowConfigWarning) {
+		// In this case if the mod we just loaded a compressed modpack, we can't edit or modify files without decompressing it.
+		if (Options.devMode && Options.allowConfigWarning && !isZipMod) {
 			var lib:ModsFolderLibrary;
 			for (e in Paths.assetsTree.libraries) if ((lib = cast AssetsLibraryList.getCleanLibrary(e)) is ModsFolderLibrary
 				&& lib.modName == ModsFolder.currentModFolder)

source/funkin/backend/system/macros/Macros.hx

@@ -69,6 +69,7 @@ class Macros {
 		final fields:Array<Field> = Context.getBuildFields(), pos:Position = Context.currentPos();
 
 		fields.push({name: 'tag', access: [APublic], pos: pos, kind: FVar(macro :funkin.backend.assets.AssetSource)});
+		fields.push({name: 'isCompressed', access: [APublic], pos: pos, kind: FVar(macro :Bool, macro false)});
 
 		return fields;
 	}

source/funkin/backend/utils/CoolUtil.hx

@@ -1456,6 +1456,7 @@ final class CoolUtil
 
 		return toProperty.setValue(fromProperty.getValue());
 	}
+
 }
 
 class PropertyInfo {