db: rework 'files' to 'installed'

Make the db of installed packages more similar to index file and
reuse the code. Also rename the database file.

5 files changed, 140 insertions, 115 deletions

diff --git a/TODO b/TODO
index ee62c66..953d4a1 100644
--- a/TODO
+++ b/TODO
@@ -1,6 +1,6 @@
 - Index/pkginfo reader: same field multiple times -> memleak
 
-- Compress index and db files
+- Compress 'installed' and 'scripts'
 - Repository support:
 	- always keep local copy of index
 	- index/package fetching from URLs
diff --git a/src/apk_database.h b/src/apk_database.h
index b0b84d6..7914a03 100644
--- a/src/apk_database.h
+++ b/src/apk_database.h
@@ -96,7 +96,6 @@ void apk_db_close(struct apk_database *db);
 int apk_db_pkg_add_file(struct apk_database *db, const char *file);
 struct apk_package *apk_db_get_pkg(struct apk_database *db, csum_t sum);
 
-int apk_db_index_read(struct apk_database *db, struct apk_istream *is, int repo);
 void apk_db_index_write(struct apk_database *db, int fd);
 
 int apk_db_add_repository(apk_database_t db, apk_blob_t repository);
diff --git a/src/apk_package.h b/src/apk_package.h
index ea8ef24..f36eb6b 100644
--- a/src/apk_package.h
+++ b/src/apk_package.h
@@ -72,10 +72,14 @@ int apk_deps_format(char *buf, int size,
 		    struct apk_dependency_array *depends);
 int apk_script_type(const char *name);
 
+struct apk_package *apk_pkg_new(void);
 struct apk_package *apk_pkg_read(struct apk_database *db, const char *name);
 void apk_pkg_free(struct apk_package *pkg);
 
+int apk_pkg_add_info(struct apk_database *db, struct apk_package *pkg,
+		     char field, apk_blob_t value);
 int apk_pkg_get_state(struct apk_package *pkg);
+void apk_pkg_set_state(struct apk_database *db, struct apk_package *pkg, int state);
 int apk_pkg_add_script(struct apk_package *pkg, struct apk_istream *is,
 		       unsigned int type, unsigned int size);
 int apk_pkg_run_script(struct apk_package *pkg, int root_fd,
diff --git a/src/database.c b/src/database.c
index 13fc7b9..513b2da 100644
--- a/src/database.c
+++ b/src/database.c
@@ -55,7 +55,7 @@ static unsigned long csum_hash(apk_blob_t csum)
 {
 	/* Checksum's highest bits have the most "randomness", use that
 	 * directly as hash */
-	return *(unsigned long *)csum.ptr;
+	return *(unsigned long *) csum.ptr;
 }
 
 static const struct apk_hash_ops pkg_info_hash_ops = {
@@ -243,7 +243,24 @@ static struct apk_db_file *apk_db_file_get(struct apk_database *db,
 	return file;
 }
 
-static int apk_db_read_fdb(struct apk_database *db, struct apk_istream *is)
+static struct apk_package *apk_db_pkg_add(struct apk_database *db, struct apk_package *pkg)
+{
+	struct apk_package *idb;
+
+	idb = apk_hash_get(&db->available.packages, APK_BLOB_BUF(pkg->csum));
+	if (idb == NULL) {
+		idb = pkg;
+		pkg->id = db->pkg_id++;
+		apk_hash_insert(&db->available.packages, pkg);
+		*apk_package_array_add(&pkg->name->pkgs) = pkg;
+	} else {
+		idb->repos |= pkg->repos;
+		apk_pkg_free(pkg);
+	}
+	return idb;
+}
+
+static int apk_db_index_read(struct apk_database *db, struct apk_istream *is, int repo)
 {
 	struct apk_package *pkg = NULL;
 	struct apk_db_dir *dir = NULL;
@@ -253,8 +270,7 @@ static int apk_db_read_fdb(struct apk_database *db, struct apk_istream *is)
 
 	char buf[1024];
 	apk_blob_t l, r;
-	csum_t csum;
-	int n;
+	int n, field;
 
 	r = APK_BLOB_PTR_LEN(buf, 0);
 	while (1) {
@@ -264,38 +280,63 @@ static int apk_db_read_fdb(struct apk_database *db, struct apk_istream *is)
 		r.len += n;
 
 		while (apk_blob_splitstr(r, "\n", &l, &r)) {
-			n = l.ptr[0];
-			l.ptr++;
-			l.len--;
-			switch (n) {
-			case 'P':
-				if (apk_hexdump_parse(APK_BLOB_BUF(csum), l)) {
-					apk_error("Not a valid checksum");
-					return -1;
-				}
-				pkg = apk_db_get_pkg(db, csum);
-				if (pkg == NULL) {
-					apk_error("Package '%.*s' is installed, but not in any repository",
-						  l.len, l.ptr);
+			if (l.len < 2 || l.ptr[1] != ':') {
+				if (pkg == NULL)
+					continue;
+
+				if (repo != -1)
+					pkg->repos |= BIT(repo);
+				else
+					apk_pkg_set_state(db, pkg, APK_STATE_INSTALL);
+
+				if (apk_db_pkg_add(db, pkg) != pkg && repo == -1) {
+					apk_error("Installed database load failed");
 					return -1;
 				}
-				if (!list_hashed(&pkg->installed_pkgs_list)) {
-					db->installed.stats.packages++;
-					list_add_tail(&pkg->installed_pkgs_list, &db->installed.packages);
-				}
+				pkg = NULL;
+				continue;
+			}
+
+			/* Get field */
+			field = l.ptr[0];
+			l.ptr += 2;
+			l.len -= 2;
+
+			/* If no package, create new */
+			if (pkg == NULL) {
+				pkg = apk_pkg_new();
 				dir = NULL;
 				file_dir_node = NULL;
 				file_pkg_node = hlist_tail_ptr(&pkg->owned_files);
-				break;
-			case 'D':
-				if (pkg == NULL) {
+			}
+
+			/* Standard index line? */
+			if (apk_pkg_add_info(db, pkg, field, l) == 0)
+				continue;
+
+			if (repo != -1) {
+				apk_error("Invalid index entry '%c'", field);
+				return -1;
+			}
+
+			/* Check FDB special entries */
+			switch (field) {
+			case 'F':
+				if (pkg->name == NULL) {
 					apk_error("FDB directory entry before package entry");
 					return -1;
 				}
 				dir = apk_db_dir_get(db, l);
 				file_dir_node = hlist_tail_ptr(&dir->files);
 				break;
-			case 'F':
+			case 'M':
+				if (dir == NULL) {
+					apk_error("FDB directory metadata entry before directory entry");
+					return -1;
+				}
+				sscanf(l.ptr, "%d:%d:%o", &dir->uid, &dir->gid, &dir->mode);
+				break;
+			case 'R':
 				if (dir == NULL) {
 					apk_error("FDB file entry before directory entry");
 					return -1;
@@ -305,7 +346,7 @@ static int apk_db_read_fdb(struct apk_database *db, struct apk_istream *is)
 				file_dir_node = &file->dir_files_list.next;
 				file_pkg_node = &file->pkg_files_list.next;
 				break;
-			case 'C':
+			case 'Z':
 				if (file == NULL) {
 					apk_error("FDB checksum entry before file entry");
 					return -1;
@@ -335,35 +376,34 @@ static int apk_db_write_fdb(struct apk_database *db, int fd)
 	struct apk_db_file *file;
 	struct hlist_node *c2;
 	char buf[1024];
+	apk_blob_t blob;
 	int n;
 
 	list_for_each_entry(pkg, &db->installed.packages, installed_pkgs_list) {
-		n = 0;
-		buf[n++] = 'P';
-		n += apk_hexdump_format(sizeof(buf)-n, &buf[n],
-					APK_BLOB_BUF(pkg->csum));
-		buf[n++] = '\n';
-		if (write(fd, buf, n) != n)
-			return -1;
-		n = 0;
+		blob = apk_pkg_format_index_entry(pkg, sizeof(buf), buf);
+		if (blob.ptr)
+			write(fd, blob.ptr, blob.len - 1);
 
+		n = 0;
 		dir = NULL;
 		hlist_for_each_entry(file, c2, &pkg->owned_files, pkg_files_list) {
 			if (file->owner == NULL)
 				continue;
 
 			if (dir != file->dir) {
-				n += snprintf(&buf[n], sizeof(buf)-n,
-					     "D%s\n",
-					      file->dir->dirname);
 				dir = file->dir;
+				n += snprintf(&buf[n], sizeof(buf)-n,
+					      "F:%s\n"
+					      "M:%d:%d:%o\n",
+					      dir->dirname,
+					      dir->uid, dir->gid, dir->mode);
 			}
 
 			n += snprintf(&buf[n], sizeof(buf)-n,
-				      "F%s\n",
+				      "R:%s\n",
 				      file->filename);
 			if (csum_valid(file->csum)) {
-				n += snprintf(&buf[n], sizeof(buf)-n, "C");
+				n += snprintf(&buf[n], sizeof(buf)-n, "Z:");
 				n += apk_hexdump_format(sizeof(buf)-n, &buf[n],
 							APK_BLOB_BUF(file->csum));
 				n += snprintf(&buf[n], sizeof(buf)-n, "\n");
@@ -373,6 +413,7 @@ static int apk_db_write_fdb(struct apk_database *db, int fd)
 				return -1;
 			n = 0;
 		}
+		write(fd, "\n", 1);
 	}
 
 	return 0;
@@ -384,7 +425,7 @@ struct apk_script_header {
 	unsigned int size;
 };
 
-static int apk_db_write_scriptdb(struct apk_database *db, int fd)
+static int apk_db_scriptdb_write(struct apk_database *db, int fd)
 {
 	struct apk_package *pkg;
 	struct apk_script *script;
@@ -405,7 +446,7 @@ static int apk_db_write_scriptdb(struct apk_database *db, int fd)
 	return 0;
 }
 
-static int apk_db_read_scriptdb(struct apk_database *db, struct apk_istream *is)
+static int apk_db_scriptdb_read(struct apk_database *db, struct apk_istream *is)
 {
 	struct apk_package *pkg;
 	struct apk_script_header hdr;
@@ -470,24 +511,18 @@ static int apk_db_read_state(struct apk_database *db)
 	apk_deps_parse(db, &db->world, blob);
 	free(blob.ptr);
 
-	is = apk_istream_from_file("var/lib/apk/files");
+	is = apk_istream_from_file("var/lib/apk/installed");
 	if (is != NULL) {
-		apk_db_read_fdb(db, is);
+		apk_db_index_read(db, is, -1);
 		is->close(is);
 	}
 
 	is = apk_istream_from_file("var/lib/apk/scripts");
 	if (is != NULL) {
-		apk_db_read_scriptdb(db, is);
+		apk_db_scriptdb_read(db, is);
 		is->close(is);
 	}
 
-	blob = apk_blob_from_file("etc/apk/repositories");
-	if (!APK_BLOB_IS_NULL(blob)) {
-		apk_blob_for_each_segment(blob, "\n", apk_db_add_repository, db);
-		free(blob.ptr);
-	}
-
 	return 0;
 }
 
@@ -501,7 +536,7 @@ static int add_protected_path(void *ctx, apk_blob_t blob)
 
 int apk_db_open(struct apk_database *db, const char *root)
 {
-	apk_blob_t dirs;
+	apk_blob_t blob;
 
 	memset(db, 0, sizeof(*db));
 	apk_hash_init(&db->available.names, &pkg_name_hash_ops, 1000);
@@ -518,13 +553,24 @@ int apk_db_open(struct apk_database *db, const char *root)
 			return -1;
 		}
 	}
+
+	blob = APK_BLOB_STR("etc:-etc/init.d");
+	apk_blob_for_each_segment(blob, ":", add_protected_path, db);
+
+	if (apk_db_read_state(db) != 0)
+		return -1;
+
+	fchdir(db->root_fd);
+	blob = apk_blob_from_file("etc/apk/repositories");
+	if (!APK_BLOB_IS_NULL(blob)) {
+		apk_blob_for_each_segment(blob, "\n", apk_db_add_repository, db);
+		free(blob.ptr);
+	}
+
 	if (apk_repository != NULL)
 		apk_db_add_repository(db, APK_BLOB_STR(apk_repository));
 
-	dirs = APK_BLOB_STR("etc:-etc/init.d");
-	apk_blob_for_each_segment(dirs, ":", add_protected_path, db);
-
-	return apk_db_read_state(db);
+	return 0;
 }
 
 struct write_ctx {
@@ -549,7 +595,7 @@ static int apk_db_write_config(struct apk_database *db)
 	write(fd, buf, n);
 	close(fd);
 
-	fd = creat("var/lib/apk/files", 0600);
+	fd = creat("var/lib/apk/installed", 0600);
 	if (fd < 0)
 		return -1;
 	apk_db_write_fdb(db, fd);
@@ -558,7 +604,7 @@ static int apk_db_write_config(struct apk_database *db)
 	fd = creat("var/lib/apk/scripts", 0600);
 	if (fd < 0)
 		return -1;
-	apk_db_write_scriptdb(db, fd);
+	apk_db_scriptdb_write(db, fd);
 	close(fd);
 
 	return 0;
@@ -575,21 +621,6 @@ void apk_db_close(struct apk_database *db)
 	}
 }
 
-static void apk_db_pkg_add(struct apk_database *db, struct apk_package *pkg)
-{
-	struct apk_package *idb;
-
-	idb = apk_hash_get(&db->available.packages, APK_BLOB_BUF(pkg->csum));
-	if (idb == NULL) {
-		pkg->id = db->pkg_id++;
-		apk_hash_insert(&db->available.packages, pkg);
-		*apk_package_array_add(&pkg->name->pkgs) = pkg;
-	} else {
-		idb->repos |= pkg->repos;
-		apk_pkg_free(pkg);
-	}
-}
-
 struct apk_package *apk_db_get_pkg(struct apk_database *db, csum_t sum)
 {
 	return apk_hash_get(&db->available.packages,
@@ -608,35 +639,6 @@ int apk_db_pkg_add_file(struct apk_database *db, const char *file)
 	return TRUE;
 }
 
-int apk_db_index_read(struct apk_database *db, struct apk_istream *is, int repo)
-{
-	struct apk_package *pkg;
-	char buf[1024];
-	int n;
-	apk_blob_t l, r;
-
-	r = APK_BLOB_PTR_LEN(buf, 0);
-	while (1) {
-		n = is->read(is, &r.ptr[r.len], sizeof(buf) - r.len);
-		if (n <= 0)
-			break;
-		r.len += n;
-
-		while (apk_blob_splitstr(r, "\n\n", &l, &r)) {
-			pkg = apk_pkg_parse_index_entry(db, l);
-			if (pkg != NULL) {
-				pkg->repos |= BIT(repo);
-				apk_db_pkg_add(db, pkg);
-			}
-		}
-
-		memcpy(&buf[0], r.ptr, r.len);
-		r = APK_BLOB_PTR_LEN(buf, r.len);
-	}
-
-	return 0;
-}
-
 static int write_index_entry(apk_hash_item item, void *ctx)
 {
 	int fd = (int) ctx;
@@ -829,8 +831,7 @@ static void apk_db_purge_pkg(struct apk_database *db,
 
 		db->installed.stats.files--;
 	}
-	db->installed.stats.packages--;
-	list_del(&pkg->installed_pkgs_list);
+	apk_pkg_set_state(db, pkg, APK_STATE_NO_INSTALL);
 }
 
 int apk_db_install_pkg(struct apk_database *db,
@@ -890,8 +891,7 @@ int apk_db_install_pkg(struct apk_database *db,
 
 	bs->close(bs, csum);
 
-	db->installed.stats.packages++;
-	list_add_tail(&newpkg->installed_pkgs_list, &db->installed.packages);
+	apk_pkg_set_state(db, newpkg, APK_STATE_INSTALL);
 
 	if (memcmp(csum, newpkg->csum, sizeof(csum)) != 0)
 		apk_warning("%s-%s: checksum does not match",
diff --git a/src/package.c b/src/package.c
index 27300c7..56f0f79 100644
--- a/src/package.c
+++ b/src/package.c
@@ -25,7 +25,7 @@
 #include "apk_database.h"
 #include "apk_state.h"
 
-static struct apk_package *apk_pkg_new(void)
+struct apk_package *apk_pkg_new(void)
 {
 	struct apk_package *pkg;
 
@@ -189,8 +189,8 @@ struct read_info_ctx {
 	int has_install;
 };
 
-static int add_info(struct apk_database *db, struct apk_package *pkg,
-		    char field, apk_blob_t value)
+int apk_pkg_add_info(struct apk_database *db, struct apk_package *pkg,
+		     char field, apk_blob_t value)
 {
 	switch (field) {
 	case 'P':
@@ -220,6 +220,8 @@ static int add_info(struct apk_database *db, struct apk_package *pkg,
 	case 'I':
 		pkg->installed_size = apk_blob_uint(value, 10);
 		break;
+	default:
+		return -1;
 	}
 	return 0;
 }
@@ -250,7 +252,7 @@ static int read_info_line(void *ctx, apk_blob_t line)
 
 	for (i = 0; i < ARRAY_SIZE(fields); i++) {
 		if (strncmp(fields[i].str, l.ptr, l.len) == 0) {
-			add_info(ri->db, ri->pkg, fields[i].field, r);
+			apk_pkg_add_info(ri->db, ri->pkg, fields[i].field, r);
 			break;
 		}
 	}
@@ -309,8 +311,8 @@ static int read_info_entry(void *ctx, const struct apk_file_info *ae,
 		for (i = 0; i < ARRAY_SIZE(fields); i++) {
 			if (strcmp(fields[i].str, slash+1) == 0) {
 				apk_blob_t blob = apk_blob_from_istream(is, ae->size);
-				add_info(ri->db, ri->pkg, fields[i].field,
-					 trim(blob));
+				apk_pkg_add_info(ri->db, ri->pkg, fields[i].field,
+						 trim(blob));
 				free(blob.ptr);
 				break;
 			}
@@ -407,6 +409,25 @@ int apk_pkg_get_state(struct apk_package *pkg)
 	return APK_STATE_NO_INSTALL;
 }
 
+void apk_pkg_set_state(struct apk_database *db, struct apk_package *pkg, int state)
+{
+	switch (state) {
+	case APK_STATE_INSTALL:
+		if (!list_hashed(&pkg->installed_pkgs_list)) {
+			db->installed.stats.packages++;
+			list_add_tail(&pkg->installed_pkgs_list,
+				      &db->installed.packages);
+		}
+		break;
+	case APK_STATE_NO_INSTALL:
+		if (list_hashed(&pkg->installed_pkgs_list)) {
+			db->installed.stats.packages--;
+			list_del(&pkg->installed_pkgs_list);
+		}
+		break;
+	}
+}
+
 int apk_pkg_add_script(struct apk_package *pkg, struct apk_istream *is,
 		       unsigned int type, unsigned int size)
 {
@@ -489,7 +510,7 @@ static int parse_index_line(void *ctx, apk_blob_t line)
 	if (line.len < 3 || line.ptr[1] != ':')
 		return 0;
 
-	add_info(ri->db, ri->pkg, line.ptr[0], APK_BLOB_PTR_LEN(line.ptr+2, line.len-2));
+	apk_pkg_add_info(ri->db, ri->pkg, line.ptr[0], APK_BLOB_PTR_LEN(line.ptr+2, line.len-2));
 	return 0;
 }
 
@@ -508,7 +529,8 @@ struct apk_package *apk_pkg_parse_index_entry(struct apk_database *db, apk_blob_
 
 	if (ctx.pkg->name == NULL) {
 		apk_pkg_free(ctx.pkg);
-		printf("%.*s\n", blob.len, blob.ptr);
+		apk_error("Failed to parse index entry: %.*s",
+			  blob.len, blob.ptr);
 		ctx.pkg = NULL;
 	}