satsolver 0.16.3
|
00001 /* 00002 * Copyright (c) 2007, Novell Inc. 00003 * 00004 * This program is licensed under the BSD license, read LICENSE.BSD 00005 * for further information 00006 */ 00007 00008 /* 00009 * repo_solv.c 00010 * 00011 * Read the binary dump of a Repo and create a Repo * from it 00012 * 00013 * See 00014 * Repo *pool_addrepo_solv(Pool *pool, FILE *fp) 00015 * below 00016 * 00017 */ 00018 00019 00020 00021 #include <stdio.h> 00022 #include <stdlib.h> 00023 #include <unistd.h> 00024 #include <string.h> 00025 00026 #include "repo_solv.h" 00027 #include "util.h" 00028 00029 #include "repopack.h" 00030 #include "repopage.h" 00031 00032 #define INTERESTED_START SOLVABLE_NAME 00033 #define INTERESTED_END SOLVABLE_ENHANCES 00034 00035 #define SOLV_ERROR_NOT_SOLV 1 00036 #define SOLV_ERROR_UNSUPPORTED 2 00037 #define SOLV_ERROR_EOF 3 00038 #define SOLV_ERROR_ID_RANGE 4 00039 #define SOLV_ERROR_OVERFLOW 5 00040 #define SOLV_ERROR_CORRUPT 6 00041 00042 static Pool *mypool; /* for pool_debug... */ 00043 00044 00045 00046 /******************************************************************************* 00047 * functions to extract data from a file handle 00048 */ 00049 00050 /* 00051 * read u32 00052 */ 00053 00054 static unsigned int 00055 read_u32(Repodata *data) 00056 { 00057 int c, i; 00058 unsigned int x = 0; 00059 00060 if (data->error) 00061 return 0; 00062 for (i = 0; i < 4; i++) 00063 { 00064 c = getc(data->fp); 00065 if (c == EOF) 00066 { 00067 pool_debug(mypool, SAT_ERROR, "unexpected EOF\n"); 00068 data->error = SOLV_ERROR_EOF; 00069 return 0; 00070 } 00071 x = (x << 8) | c; 00072 } 00073 return x; 00074 } 00075 00076 00077 /* 00078 * read u8 00079 */ 00080 00081 static unsigned int 00082 read_u8(Repodata *data) 00083 { 00084 int c; 00085 00086 if (data->error) 00087 return 0; 00088 c = getc(data->fp); 00089 if (c == EOF) 00090 { 00091 pool_debug(mypool, SAT_ERROR, "unexpected EOF\n"); 00092 data->error = SOLV_ERROR_EOF; 00093 return 0; 00094 } 00095 return c; 00096 } 00097 00098 00099 /* 00100 * read Id 00101 */ 00102 00103 static Id 00104 read_id(Repodata *data, Id max) 00105 { 00106 unsigned int x = 0; 00107 int c, i; 00108 00109 if (data->error) 00110 return 0; 00111 for (i = 0; i < 5; i++) 00112 { 00113 c = getc(data->fp); 00114 if (c == EOF) 00115 { 00116 pool_debug(mypool, SAT_ERROR, "unexpected EOF\n"); 00117 data->error = SOLV_ERROR_EOF; 00118 return 0; 00119 } 00120 if (!(c & 128)) 00121 { 00122 x = (x << 7) | c; 00123 if (max && x >= max) 00124 { 00125 pool_debug(mypool, SAT_ERROR, "read_id: id too large (%u/%u)\n", x, max); 00126 data->error = SOLV_ERROR_ID_RANGE; 00127 return 0; 00128 } 00129 return x; 00130 } 00131 x = (x << 7) ^ c ^ 128; 00132 } 00133 pool_debug(mypool, SAT_ERROR, "read_id: id too long\n"); 00134 data->error = SOLV_ERROR_CORRUPT; 00135 return 0; 00136 } 00137 00138 00139 static Id * 00140 read_idarray(Repodata *data, Id max, Id *map, Id *store, Id *end) 00141 { 00142 unsigned int x = 0; 00143 int c; 00144 00145 if (data->error) 00146 return 0; 00147 for (;;) 00148 { 00149 c = getc(data->fp); 00150 if (c == EOF) 00151 { 00152 pool_debug(mypool, SAT_ERROR, "unexpected EOF\n"); 00153 data->error = SOLV_ERROR_EOF; 00154 return 0; 00155 } 00156 if ((c & 128) != 0) 00157 { 00158 x = (x << 7) ^ c ^ 128; 00159 continue; 00160 } 00161 x = (x << 6) | (c & 63); 00162 if (max && x >= max) 00163 { 00164 pool_debug(mypool, SAT_ERROR, "read_idarray: id too large (%u/%u)\n", x, max); 00165 data->error = SOLV_ERROR_ID_RANGE; 00166 return 0; 00167 } 00168 if (map) 00169 x = map[x]; 00170 if (store == end) 00171 { 00172 pool_debug(mypool, SAT_ERROR, "read_idarray: array overflow\n"); 00173 return 0; 00174 } 00175 *store++ = x; 00176 if ((c & 64) == 0) 00177 { 00178 if (x == 0) /* already have trailing zero? */ 00179 return store; 00180 if (store == end) 00181 { 00182 pool_debug(mypool, SAT_ERROR, "read_idarray: array overflow\n"); 00183 data->error = SOLV_ERROR_OVERFLOW; 00184 return 0; 00185 } 00186 *store++ = 0; 00187 return store; 00188 } 00189 x = 0; 00190 } 00191 } 00192 00193 00194 /******************************************************************************* 00195 * functions to extract data from memory 00196 */ 00197 00198 /* 00199 * read array of Ids 00200 */ 00201 00202 static inline unsigned char * 00203 data_read_id_max(unsigned char *dp, Id *ret, Id *map, int max, int *error) 00204 { 00205 Id x; 00206 dp = data_read_id(dp, &x); 00207 if (max && x >= max) 00208 { 00209 pool_debug(mypool, SAT_ERROR, "data_read_idarray: id too large (%u/%u)\n", x, max); 00210 *error = SOLV_ERROR_ID_RANGE; 00211 x = 0; 00212 } 00213 *ret = map ? map[x] : x; 00214 return dp; 00215 } 00216 00217 unsigned char * 00218 data_read_idarray(unsigned char *dp, Id **storep, Id *map, int max, int *error) 00219 { 00220 Id *store = *storep; 00221 unsigned int x = 0; 00222 int c; 00223 00224 for (;;) 00225 { 00226 c = *dp++; 00227 if ((c & 128) != 0) 00228 { 00229 x = (x << 7) ^ c ^ 128; 00230 continue; 00231 } 00232 x = (x << 6) | (c & 63); 00233 if (max && x >= max) 00234 { 00235 pool_debug(mypool, SAT_ERROR, "data_read_idarray: id too large (%u/%u)\n", x, max); 00236 *error = SOLV_ERROR_ID_RANGE; 00237 break; 00238 } 00239 *store++ = x; 00240 if ((c & 64) == 0) 00241 break; 00242 x = 0; 00243 } 00244 *store++ = 0; 00245 *storep = store; 00246 return dp; 00247 } 00248 00249 unsigned char * 00250 data_read_rel_idarray(unsigned char *dp, Id **storep, Id *map, int max, int *error, Id marker) 00251 { 00252 Id *store = *storep; 00253 Id old = 0; 00254 unsigned int x = 0; 00255 int c; 00256 00257 for (;;) 00258 { 00259 c = *dp++; 00260 if ((c & 128) != 0) 00261 { 00262 x = (x << 7) ^ c ^ 128; 00263 continue; 00264 } 00265 x = (x << 6) | (c & 63); 00266 if (x == 0) 00267 { 00268 if (!(c & 64)) 00269 break; 00270 if (marker) 00271 *store++ = marker; 00272 old = 0; 00273 continue; 00274 } 00275 x = old + (x - 1); 00276 old = x; 00277 if (max && x >= max) 00278 { 00279 pool_debug(mypool, SAT_ERROR, "data_read_rel_idarray: id too large (%u/%u)\n", x, max); 00280 *error = SOLV_ERROR_ID_RANGE; 00281 break; 00282 } 00283 *store++ = map ? map[x] : x; 00284 if (!(c & 64)) 00285 break; 00286 x = 0; 00287 } 00288 *store++ = 0; 00289 *storep = store; 00290 return dp; 00291 } 00292 00293 00294 00295 00296 /******************************************************************************* 00297 * functions to add data to our incore memory space 00298 */ 00299 00300 #define INCORE_ADD_CHUNK 8192 00301 #define DATA_READ_CHUNK 8192 00302 00303 static void 00304 incore_add_id(Repodata *data, Id x) 00305 { 00306 unsigned char *dp; 00307 /* make sure we have at least 5 bytes free */ 00308 if (data->incoredatafree < 5) 00309 { 00310 data->incoredata = sat_realloc(data->incoredata, data->incoredatalen + INCORE_ADD_CHUNK); 00311 data->incoredatafree = INCORE_ADD_CHUNK; 00312 } 00313 dp = data->incoredata + data->incoredatalen; 00314 if (x < 0) 00315 abort(); 00316 if (x >= (1 << 14)) 00317 { 00318 if (x >= (1 << 28)) 00319 *dp++ = (x >> 28) | 128; 00320 if (x >= (1 << 21)) 00321 *dp++ = (x >> 21) | 128; 00322 *dp++ = (x >> 14) | 128; 00323 } 00324 if (x >= (1 << 7)) 00325 *dp++ = (x >> 7) | 128; 00326 *dp++ = x & 127; 00327 data->incoredatafree -= dp - (data->incoredata + data->incoredatalen); 00328 data->incoredatalen = dp - data->incoredata; 00329 } 00330 00331 static void 00332 incore_add_blob(Repodata *data, unsigned char *buf, int len) 00333 { 00334 if (data->incoredatafree < len) 00335 { 00336 data->incoredata = sat_realloc(data->incoredata, data->incoredatalen + INCORE_ADD_CHUNK + len); 00337 data->incoredatafree = INCORE_ADD_CHUNK + len; 00338 } 00339 memcpy(data->incoredata + data->incoredatalen, buf, len); 00340 data->incoredatafree -= len; 00341 data->incoredatalen += len; 00342 } 00343 00344 static void 00345 incore_map_idarray(Repodata *data, unsigned char *dp, Id *map, Id max) 00346 { 00347 /* We have to map the IDs, which might also change 00348 the necessary number of bytes, so we can't just copy 00349 over the blob and adjust it. */ 00350 for (;;) 00351 { 00352 Id id; 00353 int eof; 00354 dp = data_read_ideof(dp, &id, &eof); 00355 if (max && id >= max) 00356 { 00357 pool_debug(mypool, SAT_ERROR, "incore_map_idarray: id too large (%u/%u)\n", id, max); 00358 data->error = SOLV_ERROR_ID_RANGE; 00359 break; 00360 } 00361 id = map[id]; 00362 if (id >= 64) 00363 id = (id & 63) | ((id & ~63) << 1); 00364 incore_add_id(data, eof ? id : id | 64); 00365 if (eof) 00366 break; 00367 } 00368 } 00369 00370 static void 00371 incore_add_u32(Repodata *data, unsigned int x) 00372 { 00373 unsigned char *dp; 00374 /* make sure we have at least 4 bytes free */ 00375 if (data->incoredatafree < 4) 00376 { 00377 data->incoredata = sat_realloc(data->incoredata, data->incoredatalen + INCORE_ADD_CHUNK); 00378 data->incoredatafree = INCORE_ADD_CHUNK; 00379 } 00380 dp = data->incoredata + data->incoredatalen; 00381 *dp++ = x >> 24; 00382 *dp++ = x >> 16; 00383 *dp++ = x >> 8; 00384 *dp++ = x; 00385 data->incoredatafree -= 4; 00386 data->incoredatalen += 4; 00387 } 00388 00389 #if 0 00390 static void 00391 incore_add_u8(Repodata *data, unsigned int x) 00392 { 00393 unsigned char *dp; 00394 /* make sure we have at least 1 byte free */ 00395 if (data->incoredatafree < 1) 00396 { 00397 data->incoredata = sat_realloc(data->incoredata, data->incoredatalen + 1024); 00398 data->incoredatafree = 1024; 00399 } 00400 dp = data->incoredata + data->incoredatalen; 00401 *dp++ = x; 00402 data->incoredatafree--; 00403 data->incoredatalen++; 00404 } 00405 #endif 00406 00407 00408 /******************************************************************************* 00409 * our main function 00410 */ 00411 00412 /* 00413 * read repo from .solv file and add it to pool 00414 */ 00415 00416 int 00417 repo_add_solv_flags(Repo *repo, FILE *fp, int flags) 00418 { 00419 Pool *pool = repo->pool; 00420 int i, l; 00421 unsigned int numid, numrel, numdir, numsolv; 00422 unsigned int numkeys, numschemata; 00423 00424 Offset sizeid; 00425 Offset *str; /* map Id -> Offset into string space */ 00426 char *strsp; /* repo string space */ 00427 char *sp; /* pointer into string space */ 00428 Id *idmap; /* map of repo Ids to pool Ids */ 00429 Id id, type; 00430 unsigned int hashmask, h; 00431 int hh; 00432 Id *hashtbl; 00433 Id name, evr, did; 00434 int relflags; 00435 Reldep *ran; 00436 unsigned int size_idarray; 00437 Id *idarraydatap, *idarraydataend; 00438 Offset ido; 00439 Solvable *s; 00440 unsigned int solvflags; 00441 unsigned int solvversion; 00442 Repokey *keys; 00443 Id *schemadata, *schemadatap, *schemadataend; 00444 Id *schemata, key, *keyp; 00445 int nentries; 00446 int have_xdata; 00447 int maxsize, allsize; 00448 unsigned char *buf, *bufend, *dp, *dps; 00449 Id stack[3 * 5]; 00450 int keydepth; 00451 int needchunk; /* need a new chunk of data */ 00452 unsigned int now; 00453 00454 struct _Stringpool *spool; 00455 00456 Repodata *parent = 0; 00457 Repodata data; 00458 00459 now = sat_timems(0); 00460 00461 if ((flags & REPO_USE_LOADING) != 0) 00462 { 00463 /* this is a stub replace operation */ 00464 flags |= REPO_EXTEND_SOLVABLES; 00465 /* use REPO_REUSE_REPODATA hack so that the old repodata is kept */ 00466 parent = repo_add_repodata(repo, flags | REPO_REUSE_REPODATA); 00467 } 00468 00469 memset(&data, 0, sizeof(data)); 00470 data.repo = repo; 00471 data.fp = fp; 00472 repopagestore_init(&data.store); 00473 00474 mypool = pool; 00475 00476 if (read_u32(&data) != ('S' << 24 | 'O' << 16 | 'L' << 8 | 'V')) 00477 { 00478 pool_debug(pool, SAT_ERROR, "not a SOLV file\n"); 00479 return SOLV_ERROR_NOT_SOLV; 00480 } 00481 solvversion = read_u32(&data); 00482 switch (solvversion) 00483 { 00484 case SOLV_VERSION_8: 00485 break; 00486 default: 00487 pool_debug(pool, SAT_ERROR, "unsupported SOLV version\n"); 00488 return SOLV_ERROR_UNSUPPORTED; 00489 } 00490 00491 pool_freeidhashes(pool); 00492 00493 numid = read_u32(&data); 00494 numrel = read_u32(&data); 00495 numdir = read_u32(&data); 00496 numsolv = read_u32(&data); 00497 numkeys = read_u32(&data); 00498 numschemata = read_u32(&data); 00499 solvflags = read_u32(&data); 00500 00501 if (numdir && numdir < 2) 00502 { 00503 pool_debug(pool, SAT_ERROR, "bad number of dirs\n"); 00504 return SOLV_ERROR_CORRUPT; 00505 } 00506 00507 if (numrel && (flags & REPO_LOCALPOOL) != 0) 00508 { 00509 pool_debug(pool, SAT_ERROR, "relations are forbidden in a local pool\n"); 00510 return SOLV_ERROR_CORRUPT; 00511 } 00512 if (parent && numsolv) 00513 { 00514 /* make sure that we exactly replace the stub repodata */ 00515 if (parent->end - parent->start != numsolv) 00516 { 00517 pool_debug(pool, SAT_ERROR, "sub-repository solvable number does not match main repository (%d - %d)\n", parent->end - parent->start, numsolv); 00518 return SOLV_ERROR_CORRUPT; 00519 } 00520 for (i = 0; i < numsolv; i++) 00521 if (pool->solvables[parent->start + i].repo != repo) 00522 { 00523 pool_debug(pool, SAT_ERROR, "main repository contains holes\n"); 00524 return SOLV_ERROR_CORRUPT; 00525 } 00526 } 00527 00528 /******* Part 1: string IDs *****************************************/ 00529 00530 sizeid = read_u32(&data); /* size of string+Id space */ 00531 00532 /* 00533 * read strings and Ids 00534 * 00535 */ 00536 00537 00538 /* 00539 * alloc buffers 00540 */ 00541 00542 if (!(flags & REPO_LOCALPOOL)) 00543 spool = &pool->ss; 00544 else 00545 { 00546 data.localpool = 1; 00547 spool = &data.spool; 00548 spool->stringspace = sat_malloc(7); 00549 strcpy(spool->stringspace, "<NULL>"); 00550 spool->sstrings = 7; 00551 spool->nstrings = 0; 00552 } 00553 00554 /* alloc string buffer */ 00555 spool->stringspace = sat_realloc(spool->stringspace, spool->sstrings + sizeid + 1); 00556 /* alloc string offsets (Id -> Offset into string space) */ 00557 spool->strings = sat_realloc2(spool->strings, spool->nstrings + numid, sizeof(Offset)); 00558 00559 strsp = spool->stringspace; 00560 str = spool->strings; /* array of offsets into strsp, indexed by Id */ 00561 00562 /* point to _BEHIND_ already allocated string/Id space */ 00563 strsp += spool->sstrings; 00564 00565 00566 /* 00567 * read new repo at end of pool 00568 */ 00569 00570 if ((solvflags & SOLV_FLAG_PREFIX_POOL) == 0) 00571 { 00572 if (sizeid && fread(strsp, sizeid, 1, fp) != 1) 00573 { 00574 pool_debug(pool, SAT_ERROR, "read error while reading strings\n"); 00575 return SOLV_ERROR_EOF; 00576 } 00577 } 00578 else 00579 { 00580 unsigned int pfsize = read_u32(&data); 00581 char *prefix = sat_malloc(pfsize); 00582 char *pp = prefix; 00583 char *old_str = 0; 00584 char *dest = strsp; 00585 int freesp = sizeid; 00586 00587 if (pfsize && fread(prefix, pfsize, 1, fp) != 1) 00588 { 00589 pool_debug(pool, SAT_ERROR, "read error while reading strings\n"); 00590 sat_free(prefix); 00591 return SOLV_ERROR_EOF; 00592 } 00593 for (i = 1; i < numid; i++) 00594 { 00595 int same = (unsigned char)*pp++; 00596 size_t len = strlen(pp) + 1; 00597 freesp -= same + len; 00598 if (freesp < 0) 00599 { 00600 pool_debug(pool, SAT_ERROR, "overflow while expanding strings\n"); 00601 sat_free(prefix); 00602 return SOLV_ERROR_OVERFLOW; 00603 } 00604 if (same) 00605 memcpy(dest, old_str, same); 00606 memcpy(dest + same, pp, len); 00607 pp += len; 00608 old_str = dest; 00609 dest += same + len; 00610 } 00611 sat_free(prefix); 00612 if (freesp != 0) 00613 { 00614 pool_debug(pool, SAT_ERROR, "expanding strings size mismatch\n"); 00615 return SOLV_ERROR_CORRUPT; 00616 } 00617 } 00618 strsp[sizeid] = 0; /* make string space \0 terminated */ 00619 sp = strsp; 00620 00621 if ((flags & REPO_LOCALPOOL) != 0) 00622 { 00623 /* no shared pool, thus no idmap and no unification */ 00624 idmap = 0; 00625 spool->nstrings = numid; 00626 str[0] = 0; 00627 if (*sp) 00628 { 00629 /* we need the '' for directories */ 00630 pool_debug(pool, SAT_ERROR, "store strings don't start with ''\n"); 00631 return SOLV_ERROR_CORRUPT; 00632 } 00633 for (i = 1; i < spool->nstrings; i++) 00634 { 00635 if (sp >= strsp + sizeid) 00636 { 00637 pool_debug(pool, SAT_ERROR, "not enough strings\n"); 00638 return SOLV_ERROR_OVERFLOW; 00639 } 00640 str[i] = sp - spool->stringspace; 00641 sp += strlen(sp) + 1; 00642 } 00643 spool->sstrings = sp - spool->stringspace; 00644 } 00645 else 00646 { 00647 00648 /* alloc id map for name and rel Ids. this maps ids in the solv files 00649 * to the ids in our pool */ 00650 idmap = sat_calloc(numid + numrel, sizeof(Id)); 00651 00652 /* 00653 * build hashes for all read strings 00654 * 00655 */ 00656 00657 hashmask = mkmask(spool->nstrings + numid); 00658 00659 #if 0 00660 POOL_DEBUG(SAT_DEBUG_STATS, "read %d strings\n", numid); 00661 POOL_DEBUG(SAT_DEBUG_STATS, "string hash buckets: %d\n", hashmask + 1); 00662 #endif 00663 00664 /* 00665 * create hashtable with strings already in pool 00666 */ 00667 00668 hashtbl = sat_calloc(hashmask + 1, sizeof(Id)); 00669 for (i = 1; i < spool->nstrings; i++) /* leave out our dummy zero id */ 00670 { 00671 h = strhash(spool->stringspace + spool->strings[i]) & hashmask; 00672 hh = HASHCHAIN_START; 00673 while (hashtbl[h]) 00674 h = HASHCHAIN_NEXT(h, hh, hashmask); 00675 hashtbl[h] = i; 00676 } 00677 00678 /* 00679 * run over string space, calculate offsets 00680 * 00681 * build id map (maps solv Id -> pool Id) 00682 */ 00683 00684 for (i = 1; i < numid; i++) 00685 { 00686 if (sp >= strsp + sizeid) 00687 { 00688 sat_free(hashtbl); 00689 sat_free(idmap); 00690 pool_debug(pool, SAT_ERROR, "not enough strings %d %d\n", i, numid); 00691 return SOLV_ERROR_OVERFLOW; 00692 } 00693 if (!*sp) /* empty string */ 00694 { 00695 idmap[i] = ID_EMPTY; 00696 sp++; 00697 continue; 00698 } 00699 00700 /* find hash slot */ 00701 h = strhash(sp) & hashmask; 00702 hh = HASHCHAIN_START; 00703 for (;;) 00704 { 00705 id = hashtbl[h]; 00706 if (id == 0) 00707 break; 00708 if (!strcmp(spool->stringspace + spool->strings[id], sp)) 00709 break; /* existing string */ 00710 h = HASHCHAIN_NEXT(h, hh, hashmask); 00711 } 00712 00713 /* length == offset to next string */ 00714 l = strlen(sp) + 1; 00715 if (id == ID_NULL) /* end of hash chain -> new string */ 00716 { 00717 id = spool->nstrings++; 00718 hashtbl[h] = id; 00719 str[id] = spool->sstrings; /* save Offset */ 00720 if (sp != spool->stringspace + spool->sstrings) /* not at end-of-buffer */ 00721 memmove(spool->stringspace + spool->sstrings, sp, l); /* append to pool buffer */ 00722 spool->sstrings += l; 00723 } 00724 idmap[i] = id; /* repo relative -> pool relative */ 00725 sp += l; /* next string */ 00726 } 00727 sat_free(hashtbl); 00728 } 00729 pool_shrink_strings(pool); /* vacuum */ 00730 00731 00732 /******* Part 2: Relation IDs ***************************************/ 00733 00734 /* 00735 * read RelDeps 00736 * 00737 */ 00738 00739 if (numrel) 00740 { 00741 /* extend rels */ 00742 pool->rels = sat_realloc2(pool->rels, pool->nrels + numrel, sizeof(Reldep)); 00743 ran = pool->rels; 00744 00745 hashmask = mkmask(pool->nrels + numrel); 00746 #if 0 00747 POOL_DEBUG(SAT_DEBUG_STATS, "read %d rels\n", numrel); 00748 POOL_DEBUG(SAT_DEBUG_STATS, "rel hash buckets: %d\n", hashmask + 1); 00749 #endif 00750 /* 00751 * prep hash table with already existing RelDeps 00752 */ 00753 00754 hashtbl = sat_calloc(hashmask + 1, sizeof(Id)); 00755 for (i = 1; i < pool->nrels; i++) 00756 { 00757 h = relhash(ran[i].name, ran[i].evr, ran[i].flags) & hashmask; 00758 hh = HASHCHAIN_START; 00759 while (hashtbl[h]) 00760 h = HASHCHAIN_NEXT(h, hh, hashmask); 00761 hashtbl[h] = i; 00762 } 00763 00764 /* 00765 * read RelDeps from repo 00766 */ 00767 00768 for (i = 0; i < numrel; i++) 00769 { 00770 name = read_id(&data, i + numid); /* read (repo relative) Ids */ 00771 evr = read_id(&data, i + numid); 00772 relflags = read_u8(&data); 00773 name = idmap[name]; /* map to (pool relative) Ids */ 00774 evr = idmap[evr]; 00775 h = relhash(name, evr, relflags) & hashmask; 00776 hh = HASHCHAIN_START; 00777 for (;;) 00778 { 00779 id = hashtbl[h]; 00780 if (id == ID_NULL) /* end of hash chain */ 00781 break; 00782 if (ran[id].name == name && ran[id].evr == evr && ran[id].flags == relflags) 00783 break; 00784 h = HASHCHAIN_NEXT(h, hh, hashmask); 00785 } 00786 if (id == ID_NULL) /* new RelDep */ 00787 { 00788 id = pool->nrels++; 00789 hashtbl[h] = id; 00790 ran[id].name = name; 00791 ran[id].evr = evr; 00792 ran[id].flags = relflags; 00793 } 00794 idmap[i + numid] = MAKERELDEP(id); /* fill Id map */ 00795 } 00796 sat_free(hashtbl); 00797 pool_shrink_rels(pool); /* vacuum */ 00798 } 00799 00800 00801 /******* Part 3: Dirs ***********************************************/ 00802 if (numdir) 00803 { 00804 data.dirpool.dirs = sat_malloc2(numdir, sizeof(Id)); 00805 data.dirpool.ndirs = numdir; 00806 data.dirpool.dirs[0] = 0; /* dir 0: virtual root */ 00807 data.dirpool.dirs[1] = 1; /* dir 1: / */ 00808 for (i = 2; i < numdir; i++) 00809 { 00810 id = read_id(&data, i + numid); 00811 if (id >= numid) 00812 data.dirpool.dirs[i] = -(id - numid); 00813 else if (idmap) 00814 data.dirpool.dirs[i] = idmap[id]; 00815 else 00816 data.dirpool.dirs[i] = id; 00817 } 00818 } 00819 00820 /******* Part 4: Keys ***********************************************/ 00821 00822 keys = sat_calloc(numkeys, sizeof(*keys)); 00823 /* keys start at 1 */ 00824 for (i = 1; i < numkeys; i++) 00825 { 00826 id = read_id(&data, numid); 00827 if (idmap) 00828 id = idmap[id]; 00829 else if ((flags & REPO_LOCALPOOL) != 0) 00830 id = str2id(pool, stringpool_id2str(spool, id), 1); 00831 type = read_id(&data, numid); 00832 if (idmap) 00833 type = idmap[type]; 00834 else if ((flags & REPO_LOCALPOOL) != 0) 00835 type = str2id(pool, stringpool_id2str(spool, type), 1); 00836 if (type < REPOKEY_TYPE_VOID || type > REPOKEY_TYPE_FLEXARRAY) 00837 { 00838 pool_debug(pool, SAT_ERROR, "unsupported data type '%s'\n", id2str(pool, type)); 00839 data.error = SOLV_ERROR_UNSUPPORTED; 00840 type = REPOKEY_TYPE_VOID; 00841 } 00842 keys[i].name = id; 00843 keys[i].type = type; 00844 keys[i].size = read_id(&data, keys[i].type == REPOKEY_TYPE_CONSTANTID ? numid + numrel : 0); 00845 keys[i].storage = read_id(&data, 0); 00846 if (id >= SOLVABLE_NAME && id <= RPM_RPMDBID) 00847 keys[i].storage = KEY_STORAGE_SOLVABLE; 00848 else if (keys[i].storage == KEY_STORAGE_SOLVABLE) 00849 keys[i].storage = KEY_STORAGE_INCORE; 00850 if (keys[i].type == REPOKEY_TYPE_CONSTANTID) 00851 { 00852 if (idmap) 00853 keys[i].size = idmap[keys[i].size]; 00854 else if ((flags & REPO_LOCALPOOL) != 0) 00855 keys[i].size = str2id(pool, stringpool_id2str(spool, keys[i].size), 1); 00856 } 00857 #if 0 00858 fprintf(stderr, "key %d %s %s %d %d\n", i, id2str(pool,id), id2str(pool, keys[i].type), 00859 keys[i].size, keys[i].storage); 00860 #endif 00861 } 00862 00863 have_xdata = parent ? 1 : 0; 00864 for (i = 1; i < numkeys; i++) 00865 if (keys[i].storage == KEY_STORAGE_INCORE || keys[i].storage == KEY_STORAGE_VERTICAL_OFFSET) 00866 have_xdata = 1; 00867 00868 data.keys = keys; 00869 data.nkeys = numkeys; 00870 for (i = 1; i < numkeys; i++) 00871 { 00872 id = keys[i].name; 00873 data.keybits[(id >> 3) & (sizeof(data.keybits) - 1)] |= 1 << (id & 7); 00874 } 00875 00876 /******* Part 5: Schemata ********************************************/ 00877 00878 id = read_id(&data, 0); 00879 schemadata = sat_calloc(id + 1, sizeof(Id)); 00880 schemadatap = schemadata + 1; 00881 schemadataend = schemadatap + id; 00882 schemata = sat_calloc(numschemata, sizeof(Id)); 00883 for (i = 1; i < numschemata; i++) 00884 { 00885 schemata[i] = schemadatap - schemadata; 00886 schemadatap = read_idarray(&data, numid, 0, schemadatap, schemadataend); 00887 #if 0 00888 Id *sp = schemadata + schemata[i]; 00889 fprintf(stderr, "schema %d:", i); 00890 for (; *sp; sp++) 00891 fprintf(stderr, " %d", *sp); 00892 fprintf(stderr, "\n"); 00893 #endif 00894 } 00895 data.schemata = schemata; 00896 data.nschemata = numschemata; 00897 data.schemadata = schemadata; 00898 data.schemadatalen = schemadataend - data.schemadata; 00899 00900 /******* Part 6: Data ********************************************/ 00901 00902 idarraydatap = idarraydataend = 0; 00903 size_idarray = 0; 00904 00905 maxsize = read_id(&data, 0); 00906 allsize = read_id(&data, 0); 00907 maxsize += 5; /* so we can read the next schema of an array */ 00908 if (maxsize > allsize) 00909 maxsize = allsize; 00910 00911 buf = sat_calloc(maxsize + DATA_READ_CHUNK + 4, 1); /* 4 extra bytes to detect overflows */ 00912 bufend = buf; 00913 dp = buf; 00914 00915 l = maxsize; 00916 if (l < DATA_READ_CHUNK) 00917 l = DATA_READ_CHUNK; 00918 if (l > allsize) 00919 l = allsize; 00920 if (!l || fread(buf, l, 1, data.fp) != 1) 00921 { 00922 pool_debug(mypool, SAT_ERROR, "unexpected EOF\n"); 00923 data.error = SOLV_ERROR_EOF; 00924 id = 0; 00925 } 00926 else 00927 { 00928 bufend = buf + l; 00929 allsize -= l; 00930 dp = data_read_id_max(dp, &id, 0, numschemata, &data.error); 00931 } 00932 00933 incore_add_id(&data, 0); /* XXX? */ 00934 incore_add_id(&data, id); 00935 keyp = schemadata + schemata[id]; 00936 data.mainschema = id; 00937 for (i = 0; keyp[i]; i++) 00938 ; 00939 if (i) 00940 data.mainschemaoffsets = sat_calloc(i, sizeof(Id)); 00941 00942 nentries = 0; 00943 keydepth = 0; 00944 s = 0; 00945 needchunk = 1; 00946 for(;;) 00947 { 00948 /* make sure we have enough room */ 00949 if (keydepth == 0 || needchunk) 00950 { 00951 int left = bufend - dp; 00952 /* read data chunk to dp */ 00953 if (data.error) 00954 break; 00955 if (left < 0) 00956 { 00957 pool_debug(mypool, SAT_ERROR, "buffer overrun\n"); 00958 data.error = SOLV_ERROR_EOF; 00959 break; 00960 } 00961 if (left < maxsize) 00962 { 00963 if (left) 00964 memmove(buf, dp, left); 00965 l = maxsize - left; 00966 if (l < DATA_READ_CHUNK) 00967 l = DATA_READ_CHUNK; 00968 if (l > allsize) 00969 l = allsize; 00970 if (l && fread(buf + left, l, 1, data.fp) != 1) 00971 { 00972 pool_debug(mypool, SAT_ERROR, "unexpected EOF\n"); 00973 data.error = SOLV_ERROR_EOF; 00974 break; 00975 } 00976 allsize -= l; 00977 left += l; 00978 bufend = buf + left; 00979 if (allsize + left < maxsize) 00980 maxsize = allsize + left; 00981 dp = buf; 00982 } 00983 needchunk = 0; 00984 } 00985 00986 key = *keyp++; 00987 #if 0 00988 printf("key %d at %d\n", key, (int)(keyp - 1 - schemadata)); 00989 #endif 00990 if (!key) 00991 { 00992 if (keydepth <= 3) 00993 needchunk = 1; 00994 if (nentries) 00995 { 00996 if (s && keydepth == 3) 00997 { 00998 s++; /* next solvable */ 00999 if (have_xdata) 01000 data.incoreoffset[(s - pool->solvables) - data.start] = data.incoredatalen; 01001 } 01002 id = stack[keydepth - 1]; 01003 if (!id) 01004 { 01005 dp = data_read_id_max(dp, &id, 0, numschemata, &data.error); 01006 incore_add_id(&data, id); 01007 } 01008 keyp = schemadata + schemata[id]; 01009 nentries--; 01010 continue; 01011 } 01012 if (!keydepth) 01013 break; 01014 --keydepth; 01015 keyp = schemadata + stack[--keydepth]; 01016 nentries = stack[--keydepth]; 01017 #if 0 01018 printf("pop flexarray %d %d\n", keydepth, nentries); 01019 #endif 01020 if (!keydepth && s) 01021 s = 0; /* back from solvables */ 01022 continue; 01023 } 01024 01025 if (keydepth == 0) 01026 data.mainschemaoffsets[keyp - 1 - (schemadata + schemata[data.mainschema])] = data.incoredatalen; 01027 01028 #if 0 01029 printf("=> %s %s %p\n", id2str(pool, keys[key].name), id2str(pool, keys[key].type), s); 01030 #endif 01031 id = keys[key].name; 01032 if (keys[key].storage == KEY_STORAGE_VERTICAL_OFFSET) 01033 { 01034 dps = dp; 01035 dp = data_skip(dp, REPOKEY_TYPE_ID); 01036 dp = data_skip(dp, REPOKEY_TYPE_ID); 01037 incore_add_blob(&data, dps, dp - dps); 01038 continue; 01039 } 01040 switch (keys[key].type) 01041 { 01042 case REPOKEY_TYPE_ID: 01043 dp = data_read_id_max(dp, &did, idmap, numid + numrel, &data.error); 01044 if (s && id == SOLVABLE_NAME) 01045 s->name = did; 01046 else if (s && id == SOLVABLE_ARCH) 01047 s->arch = did; 01048 else if (s && id == SOLVABLE_EVR) 01049 s->evr = did; 01050 else if (s && id == SOLVABLE_VENDOR) 01051 s->vendor = did; 01052 else if (keys[key].storage == KEY_STORAGE_INCORE) 01053 incore_add_id(&data, did); 01054 #if 0 01055 POOL_DEBUG(SAT_DEBUG_STATS, "%s -> %s\n", id2str(pool, id), id2str(pool, did)); 01056 #endif 01057 break; 01058 case REPOKEY_TYPE_U32: 01059 dp = data_read_u32(dp, &h); 01060 #if 0 01061 POOL_DEBUG(SAT_DEBUG_STATS, "%s -> %u\n", id2str(pool, id), h); 01062 #endif 01063 if (s && id == RPM_RPMDBID) 01064 { 01065 if (!repo->rpmdbid) 01066 repo->rpmdbid = repo_sidedata_create(repo, sizeof(Id)); 01067 repo->rpmdbid[(s - pool->solvables) - repo->start] = h; 01068 } 01069 else if (keys[key].storage == KEY_STORAGE_INCORE) 01070 incore_add_u32(&data, h); 01071 break; 01072 case REPOKEY_TYPE_IDARRAY: 01073 case REPOKEY_TYPE_REL_IDARRAY: 01074 if (!s || id < INTERESTED_START || id > INTERESTED_END) 01075 { 01076 dps = dp; 01077 dp = data_skip(dp, REPOKEY_TYPE_IDARRAY); 01078 if (keys[key].storage != KEY_STORAGE_INCORE) 01079 break; 01080 if (idmap) 01081 incore_map_idarray(&data, dps, idmap, numid); 01082 else 01083 incore_add_blob(&data, dps, dp - dps); 01084 break; 01085 } 01086 ido = idarraydatap - repo->idarraydata; 01087 if (keys[key].type == REPOKEY_TYPE_IDARRAY) 01088 dp = data_read_idarray(dp, &idarraydatap, idmap, numid + numrel, &data.error); 01089 else if (id == SOLVABLE_REQUIRES) 01090 dp = data_read_rel_idarray(dp, &idarraydatap, idmap, numid + numrel, &data.error, SOLVABLE_PREREQMARKER); 01091 else if (id == SOLVABLE_PROVIDES) 01092 dp = data_read_rel_idarray(dp, &idarraydatap, idmap, numid + numrel, &data.error, SOLVABLE_FILEMARKER); 01093 else 01094 dp = data_read_rel_idarray(dp, &idarraydatap, idmap, numid + numrel, &data.error, 0); 01095 if (idarraydatap > idarraydataend) 01096 { 01097 pool_debug(pool, SAT_ERROR, "idarray overflow\n"); 01098 data.error = SOLV_ERROR_OVERFLOW; 01099 break; 01100 } 01101 if (id == SOLVABLE_PROVIDES) 01102 s->provides = ido; 01103 else if (id == SOLVABLE_OBSOLETES) 01104 s->obsoletes = ido; 01105 else if (id == SOLVABLE_CONFLICTS) 01106 s->conflicts = ido; 01107 else if (id == SOLVABLE_REQUIRES) 01108 s->requires = ido; 01109 else if (id == SOLVABLE_RECOMMENDS) 01110 s->recommends= ido; 01111 else if (id == SOLVABLE_SUPPLEMENTS) 01112 s->supplements = ido; 01113 else if (id == SOLVABLE_SUGGESTS) 01114 s->suggests = ido; 01115 else if (id == SOLVABLE_ENHANCES) 01116 s->enhances = ido; 01117 #if 0 01118 POOL_DEBUG(SAT_DEBUG_STATS, "%s ->\n", id2str(pool, id)); 01119 for (; repo->idarraydata[ido]; ido++) 01120 POOL_DEBUG(SAT_DEBUG_STATS," %s\n", dep2str(pool, repo->idarraydata[ido])); 01121 #endif 01122 break; 01123 case REPOKEY_TYPE_FIXARRAY: 01124 case REPOKEY_TYPE_FLEXARRAY: 01125 if (!keydepth) 01126 needchunk = 1; 01127 if (keydepth == sizeof(stack)/sizeof(*stack)) 01128 { 01129 pool_debug(pool, SAT_ERROR, "array stack overflow\n"); 01130 data.error = SOLV_ERROR_CORRUPT; 01131 break; 01132 } 01133 stack[keydepth++] = nentries; 01134 stack[keydepth++] = keyp - schemadata; 01135 stack[keydepth++] = 0; 01136 dp = data_read_id(dp, &nentries); 01137 incore_add_id(&data, nentries); 01138 if (!nentries) 01139 { 01140 /* zero size array? */ 01141 keydepth -= 2; 01142 nentries = stack[--keydepth]; 01143 break; 01144 } 01145 if (keydepth == 3 && id == REPOSITORY_SOLVABLES) 01146 { 01147 /* horray! here come the solvables */ 01148 if (nentries != numsolv) 01149 { 01150 pool_debug(pool, SAT_ERROR, "inconsistent number of solvables: %d %d\n", nentries, numsolv); 01151 data.error = SOLV_ERROR_CORRUPT; 01152 break; 01153 } 01154 if (idarraydatap) 01155 { 01156 pool_debug(pool, SAT_ERROR, "more than one solvable block\n"); 01157 data.error = SOLV_ERROR_CORRUPT; 01158 break; 01159 } 01160 if (parent) 01161 s = pool_id2solvable(pool, parent->start); 01162 else 01163 s = pool_id2solvable(pool, repo_add_solvable_block(repo, numsolv)); 01164 data.start = s - pool->solvables; 01165 data.end = data.start + numsolv; 01166 repodata_extend_block(&data, data.start, numsolv); 01167 for (i = 1; i < numkeys; i++) 01168 { 01169 id = keys[i].name; 01170 if ((keys[i].type == REPOKEY_TYPE_IDARRAY || keys[i].type == REPOKEY_TYPE_REL_IDARRAY) 01171 && id >= INTERESTED_START && id <= INTERESTED_END) 01172 size_idarray += keys[i].size; 01173 } 01174 /* allocate needed space in repo */ 01175 /* we add maxsize because it is an upper limit for all idarrays, thus we can't overflow */ 01176 repo_reserve_ids(repo, 0, size_idarray + maxsize + 1); 01177 idarraydatap = repo->idarraydata + repo->idarraysize; 01178 repo->idarraysize += size_idarray; 01179 idarraydataend = idarraydatap + size_idarray; 01180 repo->lastoff = 0; 01181 if (have_xdata) 01182 data.incoreoffset[(s - pool->solvables) - data.start] = data.incoredatalen; 01183 } 01184 nentries--; 01185 dp = data_read_id_max(dp, &id, 0, numschemata, &data.error); 01186 incore_add_id(&data, id); 01187 if (keys[key].type == REPOKEY_TYPE_FIXARRAY) 01188 { 01189 if (!id) 01190 { 01191 pool_debug(pool, SAT_ERROR, "illegal fixarray\n"); 01192 data.error = SOLV_ERROR_CORRUPT; 01193 } 01194 stack[keydepth - 1] = id; 01195 } 01196 keyp = schemadata + schemata[id]; 01197 break; 01198 default: 01199 dps = dp; 01200 dp = data_skip(dp, keys[key].type); 01201 if (keys[key].storage == KEY_STORAGE_INCORE) 01202 incore_add_blob(&data, dps, dp - dps); 01203 break; 01204 } 01205 } 01206 /* should shrink idarraydata again */ 01207 01208 if (keydepth) 01209 { 01210 pool_debug(pool, SAT_ERROR, "unexpected EOF, depth = %d\n", keydepth); 01211 data.error = SOLV_ERROR_CORRUPT; 01212 } 01213 if (!data.error) 01214 { 01215 if (dp > bufend) 01216 { 01217 pool_debug(mypool, SAT_ERROR, "buffer overrun\n"); 01218 data.error = SOLV_ERROR_EOF; 01219 } 01220 } 01221 sat_free(buf); 01222 01223 if (data.error) 01224 { 01225 /* free solvables */ 01226 repo_free_solvable_block(repo, data.start, data.end - data.start, 1); 01227 /* free id array */ 01228 repo->idarraysize -= size_idarray; 01229 /* free incore data */ 01230 data.incoredata = sat_free(data.incoredata); 01231 data.incoredatalen = data.incoredatafree = 0; 01232 } 01233 01234 if (data.incoredatafree) 01235 { 01236 /* shrink excess size */ 01237 data.incoredata = sat_realloc(data.incoredata, data.incoredatalen); 01238 data.incoredatafree = 0; 01239 } 01240 01241 for (i = 1; i < numkeys; i++) 01242 if (keys[i].storage == KEY_STORAGE_VERTICAL_OFFSET) 01243 break; 01244 if (i < numkeys && !data.error) 01245 { 01246 Id fileoffset = 0; 01247 unsigned int pagesize; 01248 01249 /* we have vertical data, make it available */ 01250 data.verticaloffset = sat_calloc(numkeys, sizeof(Id)); 01251 for (i = 1; i < numkeys; i++) 01252 if (keys[i].storage == KEY_STORAGE_VERTICAL_OFFSET) 01253 { 01254 data.verticaloffset[i] = fileoffset; 01255 fileoffset += keys[i].size; 01256 } 01257 data.lastverticaloffset = fileoffset; 01258 pagesize = read_u32(&data); 01259 data.error = repopagestore_read_or_setup_pages(&data.store, data.fp, pagesize, fileoffset); 01260 } 01261 else 01262 { 01263 /* no longer needed */ 01264 data.fp = 0; 01265 } 01266 sat_free(idmap); 01267 mypool = 0; 01268 01269 if (data.error) 01270 { 01271 /* XXX: free repodata? */ 01272 return data.error; 01273 } 01274 01275 if (parent) 01276 { 01277 /* overwrite stub repodata */ 01278 repodata_freedata(parent); 01279 *parent = data; 01280 } 01281 else 01282 { 01283 /* make it available as new repodata */ 01284 repo->repodata = sat_realloc2(repo->repodata, repo->nrepodata + 1, sizeof(data)); 01285 repo->repodata[repo->nrepodata++] = data; 01286 } 01287 01288 /* create stub repodata entries for all external */ 01289 if (!(flags & SOLV_ADD_NO_STUBS) && !parent) 01290 { 01291 for (key = 1 ; key < data.nkeys; key++) 01292 if (data.keys[key].name == REPOSITORY_EXTERNAL && data.keys[key].type == REPOKEY_TYPE_FLEXARRAY) 01293 break; 01294 if (key < data.nkeys) 01295 repodata_create_stubs(repo->repodata + (repo->nrepodata - 1)); 01296 } 01297 01298 POOL_DEBUG(SAT_DEBUG_STATS, "repo_add_solv took %d ms\n", sat_timems(now)); 01299 POOL_DEBUG(SAT_DEBUG_STATS, "repo size: %d solvables\n", repo->nsolvables); 01300 POOL_DEBUG(SAT_DEBUG_STATS, "repo memory used: %d K incore, %d K idarray\n", data.incoredatalen/1024, repo->idarraysize / (int)(1024/sizeof(Id))); 01301 return 0; 01302 } 01303 01304 int 01305 repo_add_solv(Repo *repo, FILE *fp) 01306 { 01307 return repo_add_solv_flags(repo, fp, 0); 01308 }