RuneWiki Difference between revisions of "Old Engine/File Store"

Difference between revisions of "Old Engine/File Store"

From RuneWiki
Line 1: Line 1:
This file format, .dat/.idx, was introduced in [https://runescape.fandom.com/wiki/Update:New_game_update_system revision 234].
This file format, .dat/.idx0-4, was introduced in [https://runescape.fandom.com/wiki/Update:New_game_update_system revision 234]. It was replaced by .dat2/idx0-255 after [https://runescape.fandom.com/wiki/Update:Game_engine_upgraded! revision 400]. It allowed Jagex to push updates without forcing clients to redownload everything compared to using the original [[Old Engine/File Archive|File Archive]] format alone. File archives were moved inside of .idx0.


It allowed Jagex to push updates without forcing clients to redownload everything.
.idx files are essentially lookup tables for the .dat store.
 
A cache entry uses 6 bytes in the respective idx file:
 
* 3 bytes for the file size
 
* 3 bytes for the starting dat sector
 
A dat sector can be up to 520 bytes long, 8 bytes for the header and the remaining bytes for the cache entry data.
 
A sector header looks like:
 
* 2 bytes defining the file ID
* 2 bytes defining the file part (how many have been read)
* 3 bytes defining the next sector (seek here)
* 1 byte defining the sector store (.idx file used)
 
Example:<syntaxhighlight lang="javascript">
async read(file) {
let temp = new Uint8Array(520);
 
this.idx.seek(file * 6);
await this.idx.read(temp, 0, 6);
 
let size = ((temp[0] & 0xFF) << 16) + ((temp[1] & 0xFF) << 8) + (temp[2] & 0xFF);
let sector = ((temp[3] & 0xFF) << 16) + ((temp[4] & 0xFF) << 8) + (temp[5] & 0xFF);
 
if (size > this.maxFileSize) {
return new Uint8Array();
}
 
if (sector <= 0 || sector > await this.dat.length() / 520) {
return new Uint8Array();
}
 
let data = new Uint8Array(size);
let position = 0;
 
for (let part = 0; position < size; part++) {
if (sector === 0) {
return new Uint8Array();
}
 
let available = size - position;
if (available > 512) {
available = 512;
}
 
this.dat.seek(sector * 520);
await this.dat.read(temp, 0, available + 8);
 
let sectorFile = ((temp[0] & 0xFF) << 8) + (temp[1] & 0xFF);
let sectorPart = ((temp[2] & 0xFF) << 8) + (temp[3] & 0xFF);
let nextSector = ((temp[4] & 0xFF) << 16) + ((temp[5] & 0xFF) << 8) + (temp[6] & 0xFF);
let sectorStore = temp[7] & 0xFF;
 
if (sectorFile !== file || sectorPart !== part || sectorStore !== this.store) {
return new Uint8Array();
}
 
if (nextSector < 0 || nextSector > await this.dat.length() / 520) {
return new Uint8Array();
}
 
for (let i = 0; i < available; i++) {
data[position++] = temp[i + 8];
}
 
sector = nextSector;
}
return data;
}
</syntaxhighlight>

Revision as of 00:01, 8 September 2021

This file format, .dat/.idx0-4, was introduced in revision 234. It was replaced by .dat2/idx0-255 after revision 400. It allowed Jagex to push updates without forcing clients to redownload everything compared to using the original File Archive format alone. File archives were moved inside of .idx0.

.idx files are essentially lookup tables for the .dat store.

A cache entry uses 6 bytes in the respective idx file:

  • 3 bytes for the file size
  • 3 bytes for the starting dat sector

A dat sector can be up to 520 bytes long, 8 bytes for the header and the remaining bytes for the cache entry data.

A sector header looks like:

  • 2 bytes defining the file ID
  • 2 bytes defining the file part (how many have been read)
  • 3 bytes defining the next sector (seek here)
  • 1 byte defining the sector store (.idx file used)

Example:

async read(file) {
	let temp = new Uint8Array(520);

	this.idx.seek(file * 6);
	await this.idx.read(temp, 0, 6);

	let size = ((temp[0] & 0xFF) << 16) + ((temp[1] & 0xFF) << 8) + (temp[2] & 0xFF);
	let sector = ((temp[3] & 0xFF) << 16) + ((temp[4] & 0xFF) << 8) + (temp[5] & 0xFF);

	if (size > this.maxFileSize) {
		return new Uint8Array();
	}

	if (sector <= 0 || sector > await this.dat.length() / 520) {
		return new Uint8Array();
	}

	let data = new Uint8Array(size);
	let position = 0;

	for (let part = 0; position < size; part++) {
		if (sector === 0) {
			return new Uint8Array();
		}

		let available = size - position;
		if (available > 512) {
			available = 512;
		}

		this.dat.seek(sector * 520);
		await this.dat.read(temp, 0, available + 8);

		let sectorFile = ((temp[0] & 0xFF) << 8) + (temp[1] & 0xFF);
		let sectorPart = ((temp[2] & 0xFF) << 8) + (temp[3] & 0xFF);
		let nextSector = ((temp[4] & 0xFF) << 16) + ((temp[5] & 0xFF) << 8) + (temp[6] & 0xFF);
		let sectorStore = temp[7] & 0xFF;

		if (sectorFile !== file || sectorPart !== part || sectorStore !== this.store) {
			return new Uint8Array();
		}

		if (nextSector < 0 || nextSector > await this.dat.length() / 520) {
			return new Uint8Array();
		}

		for (let i = 0; i < available; i++) {
			data[position++] = temp[i + 8];
		}

		sector = nextSector;
	}
	return data;
}