View Javadoc
1   /*
2    * Copyright (C) 2008, Marek Zawirski <marek.zawirski@gmail.com> and others
3    *
4    * This program and the accompanying materials are made available under the
5    * terms of the Eclipse Distribution License v. 1.0 which is available at
6    * https://www.eclipse.org/org/documents/edl-v10.php.
7    *
8    * SPDX-License-Identifier: BSD-3-Clause
9    */
10  
11  package org.eclipse.jgit.internal.storage.file;
12  
13  import static org.eclipse.jgit.internal.storage.pack.PackWriter.NONE;
14  import static org.eclipse.jgit.lib.Constants.INFO_ALTERNATES;
15  import static org.eclipse.jgit.lib.Constants.OBJ_BLOB;
16  import static org.junit.Assert.assertEquals;
17  import static org.junit.Assert.assertFalse;
18  import static org.junit.Assert.assertNotNull;
19  import static org.junit.Assert.assertTrue;
20  import static org.junit.Assert.fail;
21  
22  import java.io.ByteArrayInputStream;
23  import java.io.ByteArrayOutputStream;
24  import java.io.File;
25  import java.io.FileOutputStream;
26  import java.io.IOException;
27  import java.text.ParseException;
28  import java.util.ArrayList;
29  import java.util.Arrays;
30  import java.util.Collections;
31  import java.util.HashSet;
32  import java.util.List;
33  import java.util.Set;
34  
35  import org.eclipse.jgit.errors.MissingObjectException;
36  import org.eclipse.jgit.internal.storage.file.PackIndex.MutableEntry;
37  import org.eclipse.jgit.internal.storage.pack.PackExt;
38  import org.eclipse.jgit.internal.storage.pack.PackWriter;
39  import org.eclipse.jgit.junit.JGitTestUtil;
40  import org.eclipse.jgit.junit.TestRepository;
41  import org.eclipse.jgit.junit.TestRepository.BranchBuilder;
42  import org.eclipse.jgit.lib.NullProgressMonitor;
43  import org.eclipse.jgit.lib.ObjectId;
44  import org.eclipse.jgit.lib.ObjectIdSet;
45  import org.eclipse.jgit.lib.ObjectInserter;
46  import org.eclipse.jgit.lib.Repository;
47  import org.eclipse.jgit.lib.Sets;
48  import org.eclipse.jgit.revwalk.DepthWalk;
49  import org.eclipse.jgit.revwalk.ObjectWalk;
50  import org.eclipse.jgit.revwalk.RevBlob;
51  import org.eclipse.jgit.revwalk.RevCommit;
52  import org.eclipse.jgit.revwalk.RevObject;
53  import org.eclipse.jgit.revwalk.RevWalk;
54  import org.eclipse.jgit.storage.pack.PackConfig;
55  import org.eclipse.jgit.storage.pack.PackStatistics;
56  import org.eclipse.jgit.test.resources.SampleDataRepositoryTestCase;
57  import org.eclipse.jgit.transport.PackParser;
58  import org.junit.After;
59  import org.junit.Before;
60  import org.junit.Test;
61  
62  public class PackWriterTest extends SampleDataRepositoryTestCase {
63  
64  	private static final List<RevObject> EMPTY_LIST_REVS = Collections
65  			.<RevObject> emptyList();
66  
67  	private static final Set<ObjectIdSet> EMPTY_ID_SET = Collections
68  			.<ObjectIdSet> emptySet();
69  
70  	private PackConfig config;
71  
72  	private PackWriter writer;
73  
74  	private ByteArrayOutputStream os;
75  
76  	private Pack pack;
77  
78  	private ObjectInserter inserter;
79  
80  	private FileRepository dst;
81  
82  	private RevBlob contentA;
83  
84  	private RevBlob contentB;
85  
86  	private RevBlob contentC;
87  
88  	private RevBlob contentD;
89  
90  	private RevBlob contentE;
91  
92  	private RevCommit c1;
93  
94  	private RevCommit c2;
95  
96  	private RevCommit c3;
97  
98  	private RevCommit c4;
99  
100 	private RevCommit c5;
101 
102 	@Override
103 	@Before
104 	public void setUp() throws Exception {
105 		super.setUp();
106 		os = new ByteArrayOutputStream();
107 		config = new PackConfig(db);
108 
109 		dst = createBareRepository();
110 		File alt = new File(dst.getObjectDatabase().getDirectory(), INFO_ALTERNATES);
111 		alt.getParentFile().mkdirs();
112 		write(alt, db.getObjectDatabase().getDirectory().getAbsolutePath() + "\n");
113 	}
114 
115 	@Override
116 	@After
117 	public void tearDown() throws Exception {
118 		if (writer != null) {
119 			writer.close();
120 			writer = null;
121 		}
122 		if (inserter != null) {
123 			inserter.close();
124 			inserter = null;
125 		}
126 		super.tearDown();
127 	}
128 
129 	/**
130 	 * Test constructor for exceptions, default settings, initialization.
131 	 *
132 	 * @throws IOException
133 	 */
134 	@Test
135 	public void testContructor() throws IOException {
136 		writer = new PackWriter(config, db.newObjectReader());
137 		assertFalse(writer.isDeltaBaseAsOffset());
138 		assertTrue(config.isReuseDeltas());
139 		assertTrue(config.isReuseObjects());
140 		assertEquals(0, writer.getObjectCount());
141 	}
142 
143 	/**
144 	 * Change default settings and verify them.
145 	 */
146 	@Test
147 	public void testModifySettings() {
148 		config.setReuseDeltas(false);
149 		config.setReuseObjects(false);
150 		config.setDeltaBaseAsOffset(false);
151 		assertFalse(config.isReuseDeltas());
152 		assertFalse(config.isReuseObjects());
153 		assertFalse(config.isDeltaBaseAsOffset());
154 
155 		writer = new PackWriter(config, db.newObjectReader());
156 		writer.setDeltaBaseAsOffset(true);
157 		assertTrue(writer.isDeltaBaseAsOffset());
158 		assertFalse(config.isDeltaBaseAsOffset());
159 	}
160 
161 	/**
162 	 * Write empty pack by providing empty sets of interesting/uninteresting
163 	 * objects and check for correct format.
164 	 *
165 	 * @throws IOException
166 	 */
167 	@Test
168 	public void testWriteEmptyPack1() throws IOException {
169 		createVerifyOpenPack(NONE, NONE, false, false);
170 
171 		assertEquals(0, writer.getObjectCount());
172 		assertEquals(0, pack.getObjectCount());
173 		assertEquals("da39a3ee5e6b4b0d3255bfef95601890afd80709", writer
174 				.computeName().name());
175 	}
176 
177 	/**
178 	 * Write empty pack by providing empty iterator of objects to write and
179 	 * check for correct format.
180 	 *
181 	 * @throws IOException
182 	 */
183 	@Test
184 	public void testWriteEmptyPack2() throws IOException {
185 		createVerifyOpenPack(EMPTY_LIST_REVS);
186 
187 		assertEquals(0, writer.getObjectCount());
188 		assertEquals(0, pack.getObjectCount());
189 	}
190 
191 	/**
192 	 * Try to pass non-existing object as uninteresting, with non-ignoring
193 	 * setting.
194 	 *
195 	 * @throws IOException
196 	 */
197 	@Test
198 	public void testNotIgnoreNonExistingObjects() throws IOException {
199 		final ObjectId nonExisting = ObjectId
200 				.fromString("0000000000000000000000000000000000000001");
201 		try {
202 			createVerifyOpenPack(NONE, haves(nonExisting), false, false);
203 			fail("Should have thrown MissingObjectException");
204 		} catch (MissingObjectException x) {
205 			// expected
206 		}
207 	}
208 
209 	/**
210 	 * Try to pass non-existing object as uninteresting, with ignoring setting.
211 	 *
212 	 * @throws IOException
213 	 */
214 	@Test
215 	public void testIgnoreNonExistingObjects() throws IOException {
216 		final ObjectId nonExisting = ObjectId
217 				.fromString("0000000000000000000000000000000000000001");
218 		createVerifyOpenPack(NONE, haves(nonExisting), false, true);
219 		// shouldn't throw anything
220 	}
221 
222 	/**
223 	 * Try to pass non-existing object as uninteresting, with ignoring setting.
224 	 * Use a repo with bitmap indexes because then PackWriter will use
225 	 * PackWriterBitmapWalker which had problems with this situation.
226 	 *
227 	 * @throws IOException
228 	 * @throws ParseException
229 	 */
230 	@Test
231 	public void testIgnoreNonExistingObjectsWithBitmaps() throws IOException,
232 			ParseException {
233 		final ObjectId nonExisting = ObjectId
234 				.fromString("0000000000000000000000000000000000000001");
235 		new GC(db).gc();
236 		createVerifyOpenPack(NONE, haves(nonExisting), false, true, true);
237 		// shouldn't throw anything
238 	}
239 
240 	/**
241 	 * Create pack basing on only interesting objects, then precisely verify
242 	 * content. No delta reuse here.
243 	 *
244 	 * @throws IOException
245 	 */
246 	@Test
247 	public void testWritePack1() throws IOException {
248 		config.setReuseDeltas(false);
249 		writeVerifyPack1();
250 	}
251 
252 	/**
253 	 * Test writing pack without object reuse. Pack content/preparation as in
254 	 * {@link #testWritePack1()}.
255 	 *
256 	 * @throws IOException
257 	 */
258 	@Test
259 	public void testWritePack1NoObjectReuse() throws IOException {
260 		config.setReuseDeltas(false);
261 		config.setReuseObjects(false);
262 		writeVerifyPack1();
263 	}
264 
265 	/**
266 	 * Create pack basing on both interesting and uninteresting objects, then
267 	 * precisely verify content. No delta reuse here.
268 	 *
269 	 * @throws IOException
270 	 */
271 	@Test
272 	public void testWritePack2() throws IOException {
273 		writeVerifyPack2(false);
274 	}
275 
276 	/**
277 	 * Test pack writing with deltas reuse, delta-base first rule. Pack
278 	 * content/preparation as in {@link #testWritePack2()}.
279 	 *
280 	 * @throws IOException
281 	 */
282 	@Test
283 	public void testWritePack2DeltasReuseRefs() throws IOException {
284 		writeVerifyPack2(true);
285 	}
286 
287 	/**
288 	 * Test pack writing with delta reuse. Delta bases referred as offsets. Pack
289 	 * configuration as in {@link #testWritePack2DeltasReuseRefs()}.
290 	 *
291 	 * @throws IOException
292 	 */
293 	@Test
294 	public void testWritePack2DeltasReuseOffsets() throws IOException {
295 		config.setDeltaBaseAsOffset(true);
296 		writeVerifyPack2(true);
297 	}
298 
299 	/**
300 	 * Test pack writing with delta reuse. Raw-data copy (reuse) is made on a
301 	 * pack with CRC32 index. Pack configuration as in
302 	 * {@link #testWritePack2DeltasReuseRefs()}.
303 	 *
304 	 * @throws IOException
305 	 */
306 	@Test
307 	public void testWritePack2DeltasCRC32Copy() throws IOException {
308 		final File packDir = db.getObjectDatabase().getPackDirectory();
309 		final PackFile crc32Pack = new PackFile(packDir,
310 				"pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.pack");
311 		final PackFile crc32Idx = new PackFile(packDir,
312 				"pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.idx");
313 		copyFile(JGitTestUtil.getTestResourceFile(
314 				"pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.idxV2"),
315 				crc32Idx);
316 		db.openPack(crc32Pack);
317 
318 		writeVerifyPack2(true);
319 	}
320 
321 	/**
322 	 * Create pack basing on fixed objects list, then precisely verify content.
323 	 * No delta reuse here.
324 	 *
325 	 * @throws IOException
326 	 * @throws MissingObjectException
327 	 *
328 	 */
329 	@Test
330 	public void testWritePack3() throws MissingObjectException, IOException {
331 		config.setReuseDeltas(false);
332 		final ObjectId forcedOrder[] = new ObjectId[] {
333 				ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"),
334 				ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"),
335 				ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"),
336 				ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327"),
337 				ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3") ,
338 				ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259") };
339 		try (RevWalk parser = new RevWalk(db)) {
340 			final RevObject forcedOrderRevs[] = new RevObject[forcedOrder.length];
341 			for (int i = 0; i < forcedOrder.length; i++)
342 				forcedOrderRevs[i] = parser.parseAny(forcedOrder[i]);
343 
344 			createVerifyOpenPack(Arrays.asList(forcedOrderRevs));
345 		}
346 
347 		assertEquals(forcedOrder.length, writer.getObjectCount());
348 		verifyObjectsOrder(forcedOrder);
349 		assertEquals("ed3f96b8327c7c66b0f8f70056129f0769323d86", writer
350 				.computeName().name());
351 	}
352 
353 	/**
354 	 * Another pack creation: basing on both interesting and uninteresting
355 	 * objects. No delta reuse possible here, as this is a specific case when we
356 	 * write only 1 commit, associated with 1 tree, 1 blob.
357 	 *
358 	 * @throws IOException
359 	 */
360 	@Test
361 	public void testWritePack4() throws IOException {
362 		writeVerifyPack4(false);
363 	}
364 
365 	/**
366 	 * Test thin pack writing: 1 blob delta base is on objects edge. Pack
367 	 * configuration as in {@link #testWritePack4()}.
368 	 *
369 	 * @throws IOException
370 	 */
371 	@Test
372 	public void testWritePack4ThinPack() throws IOException {
373 		writeVerifyPack4(true);
374 	}
375 
376 	/**
377 	 * Compare sizes of packs created using {@link #testWritePack2()} and
378 	 * {@link #testWritePack2DeltasReuseRefs()}. The pack using deltas should
379 	 * be smaller.
380 	 *
381 	 * @throws Exception
382 	 */
383 	@Test
384 	public void testWritePack2SizeDeltasVsNoDeltas() throws Exception {
385 		config.setReuseDeltas(false);
386 		config.setDeltaCompress(false);
387 		testWritePack2();
388 		final long sizePack2NoDeltas = os.size();
389 		tearDown();
390 		setUp();
391 		testWritePack2DeltasReuseRefs();
392 		final long sizePack2DeltasRefs = os.size();
393 
394 		assertTrue(sizePack2NoDeltas > sizePack2DeltasRefs);
395 	}
396 
397 	/**
398 	 * Compare sizes of packs created using
399 	 * {@link #testWritePack2DeltasReuseRefs()} and
400 	 * {@link #testWritePack2DeltasReuseOffsets()}. The pack with delta bases
401 	 * written as offsets should be smaller.
402 	 *
403 	 * @throws Exception
404 	 */
405 	@Test
406 	public void testWritePack2SizeOffsetsVsRefs() throws Exception {
407 		testWritePack2DeltasReuseRefs();
408 		final long sizePack2DeltasRefs = os.size();
409 		tearDown();
410 		setUp();
411 		testWritePack2DeltasReuseOffsets();
412 		final long sizePack2DeltasOffsets = os.size();
413 
414 		assertTrue(sizePack2DeltasRefs > sizePack2DeltasOffsets);
415 	}
416 
417 	/**
418 	 * Compare sizes of packs created using {@link #testWritePack4()} and
419 	 * {@link #testWritePack4ThinPack()}. Obviously, the thin pack should be
420 	 * smaller.
421 	 *
422 	 * @throws Exception
423 	 */
424 	@Test
425 	public void testWritePack4SizeThinVsNoThin() throws Exception {
426 		testWritePack4();
427 		final long sizePack4 = os.size();
428 		tearDown();
429 		setUp();
430 		testWritePack4ThinPack();
431 		final long sizePack4Thin = os.size();
432 
433 		assertTrue(sizePack4 > sizePack4Thin);
434 	}
435 
436 	@Test
437 	public void testDeltaStatistics() throws Exception {
438 		config.setDeltaCompress(true);
439 		// TestRepository will close repo
440 		FileRepository repo = createBareRepository();
441 		ArrayList<RevObject> blobs = new ArrayList<>();
442 		try (TestRepository<FileRepository> testRepo = new TestRepository<>(
443 				repo)) {
444 			blobs.add(testRepo.blob(genDeltableData(1000)));
445 			blobs.add(testRepo.blob(genDeltableData(1005)));
446 			try (PackWriter pw = new PackWriter(repo)) {
447 				NullProgressMonitor m = NullProgressMonitor.INSTANCE;
448 				pw.preparePack(blobs.iterator());
449 				pw.writePack(m, m, os);
450 				PackStatistics stats = pw.getStatistics();
451 				assertEquals(1, stats.getTotalDeltas());
452 				assertTrue("Delta bytes not set.",
453 						stats.byObjectType(OBJ_BLOB).getDeltaBytes() > 0);
454 			}
455 		}
456 	}
457 
458 	// Generate consistent junk data for building files that delta well
459 	private String genDeltableData(int length) {
460 		assertTrue("Generated data must have a length > 0", length > 0);
461 		char[] data = {'a', 'b', 'c', '\n'};
462 		StringBuilder builder = new StringBuilder(length);
463 		for (int i = 0; i < length; i++) {
464 			builder.append(data[i % 4]);
465 		}
466 		return builder.toString();
467 	}
468 
469 
470 	@Test
471 	public void testWriteIndex() throws Exception {
472 		config.setIndexVersion(2);
473 		writeVerifyPack4(false);
474 
475 		PackFile packFile = pack.getPackFile();
476 		PackFile indexFile = packFile.create(PackExt.INDEX);
477 
478 		// Validate that IndexPack came up with the right CRC32 value.
479 		final PackIndex idx1 = PackIndex.open(indexFile);
480 		assertTrue(idx1 instanceof PackIndexV2);
481 		assertEquals(0x4743F1E4L, idx1.findCRC32(ObjectId
482 				.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7")));
483 
484 		// Validate that an index written by PackWriter is the same.
485 		final File idx2File = new File(indexFile.getAbsolutePath() + ".2");
486 		try (FileOutputStream is = new FileOutputStream(idx2File)) {
487 			writer.writeIndex(is);
488 		}
489 		final PackIndex idx2 = PackIndex.open(idx2File);
490 		assertTrue(idx2 instanceof PackIndexV2);
491 		assertEquals(idx1.getObjectCount(), idx2.getObjectCount());
492 		assertEquals(idx1.getOffset64Count(), idx2.getOffset64Count());
493 
494 		for (int i = 0; i < idx1.getObjectCount(); i++) {
495 			final ObjectId id = idx1.getObjectId(i);
496 			assertEquals(id, idx2.getObjectId(i));
497 			assertEquals(idx1.findOffset(id), idx2.findOffset(id));
498 			assertEquals(idx1.findCRC32(id), idx2.findCRC32(id));
499 		}
500 	}
501 
502 	@Test
503 	public void testExclude() throws Exception {
504 		// TestRepository closes repo
505 		FileRepository repo = createBareRepository();
506 
507 		try (TestRepository<FileRepository> testRepo = new TestRepository<>(
508 				repo)) {
509 			BranchBuilder bb = testRepo.branch("refs/heads/master");
510 			contentA = testRepo.blob("A");
511 			c1 = bb.commit().add("f", contentA).create();
512 			testRepo.getRevWalk().parseHeaders(c1);
513 			PackIndex pf1 = writePack(repo, wants(c1), EMPTY_ID_SET);
514 			assertContent(pf1, Arrays.asList(c1.getId(), c1.getTree().getId(),
515 					contentA.getId()));
516 			contentB = testRepo.blob("B");
517 			c2 = bb.commit().add("f", contentB).create();
518 			testRepo.getRevWalk().parseHeaders(c2);
519 			PackIndex pf2 = writePack(repo, wants(c2),
520 					Sets.of((ObjectIdSet) pf1));
521 			assertContent(pf2, Arrays.asList(c2.getId(), c2.getTree().getId(),
522 					contentB.getId()));
523 		}
524 	}
525 
526 	private static void assertContent(PackIndex pi, List<ObjectId> expected) {
527 		assertEquals("Pack index has wrong size.", expected.size(),
528 				pi.getObjectCount());
529 		for (int i = 0; i < pi.getObjectCount(); i++)
530 			assertTrue(
531 					"Pack index didn't contain the expected id "
532 							+ pi.getObjectId(i),
533 					expected.contains(pi.getObjectId(i)));
534 	}
535 
536 	@Test
537 	public void testShallowIsMinimalDepth1() throws Exception {
538 		try (FileRepository repo = setupRepoForShallowFetch()) {
539 			PackIndex idx = writeShallowPack(repo, 1, wants(c2), NONE, NONE);
540 			assertContent(idx, Arrays.asList(c2.getId(), c2.getTree().getId(),
541 					contentA.getId(), contentB.getId()));
542 
543 			// Client already has blobs A and B, verify those are not packed.
544 			idx = writeShallowPack(repo, 1, wants(c5), haves(c2), shallows(c2));
545 			assertContent(idx, Arrays.asList(c5.getId(), c5.getTree().getId(),
546 					contentC.getId(), contentD.getId(), contentE.getId()));
547 		}
548 	}
549 
550 	@Test
551 	public void testShallowIsMinimalDepth2() throws Exception {
552 		try (FileRepository repo = setupRepoForShallowFetch()) {
553 			PackIndex idx = writeShallowPack(repo, 2, wants(c2), NONE, NONE);
554 			assertContent(idx,
555 					Arrays.asList(c1.getId(), c2.getId(), c1.getTree().getId(),
556 							c2.getTree().getId(), contentA.getId(),
557 							contentB.getId()));
558 
559 			// Client already has blobs A and B, verify those are not packed.
560 			idx = writeShallowPack(repo, 2, wants(c5), haves(c1, c2),
561 					shallows(c1));
562 			assertContent(idx,
563 					Arrays.asList(c4.getId(), c5.getId(), c4.getTree().getId(),
564 							c5.getTree().getId(), contentC.getId(),
565 							contentD.getId(), contentE.getId()));
566 		}
567 	}
568 
569 	@Test
570 	public void testShallowFetchShallowParentDepth1() throws Exception {
571 		try (FileRepository repo = setupRepoForShallowFetch()) {
572 			PackIndex idx = writeShallowPack(repo, 1, wants(c5), NONE, NONE);
573 			assertContent(idx, Arrays.asList(c5.getId(), c5.getTree().getId(),
574 					contentA.getId(), contentB.getId(), contentC.getId(),
575 					contentD.getId(), contentE.getId()));
576 
577 			idx = writeShallowPack(repo, 1, wants(c4), haves(c5), shallows(c5));
578 			assertContent(idx, Arrays.asList(c4.getId(), c4.getTree().getId()));
579 		}
580 	}
581 
582 	@Test
583 	public void testShallowFetchShallowParentDepth2() throws Exception {
584 		try (FileRepository repo = setupRepoForShallowFetch()) {
585 			PackIndex idx = writeShallowPack(repo, 2, wants(c5), NONE, NONE);
586 			assertContent(idx,
587 					Arrays.asList(c4.getId(), c5.getId(), c4.getTree().getId(),
588 							c5.getTree().getId(), contentA.getId(),
589 							contentB.getId(), contentC.getId(),
590 							contentD.getId(), contentE.getId()));
591 
592 			idx = writeShallowPack(repo, 2, wants(c3), haves(c4, c5),
593 					shallows(c4));
594 			assertContent(idx, Arrays.asList(c2.getId(), c3.getId(),
595 					c2.getTree().getId(), c3.getTree().getId()));
596 		}
597 	}
598 
599 	@Test
600 	public void testShallowFetchShallowAncestorDepth1() throws Exception {
601 		try (FileRepository repo = setupRepoForShallowFetch()) {
602 			PackIndex idx = writeShallowPack(repo, 1, wants(c5), NONE, NONE);
603 			assertContent(idx, Arrays.asList(c5.getId(), c5.getTree().getId(),
604 					contentA.getId(), contentB.getId(), contentC.getId(),
605 					contentD.getId(), contentE.getId()));
606 
607 			idx = writeShallowPack(repo, 1, wants(c3), haves(c5), shallows(c5));
608 			assertContent(idx, Arrays.asList(c3.getId(), c3.getTree().getId()));
609 		}
610 	}
611 
612 	@Test
613 	public void testShallowFetchShallowAncestorDepth2() throws Exception {
614 		try (FileRepository repo = setupRepoForShallowFetch()) {
615 			PackIndex idx = writeShallowPack(repo, 2, wants(c5), NONE, NONE);
616 			assertContent(idx,
617 					Arrays.asList(c4.getId(), c5.getId(), c4.getTree().getId(),
618 							c5.getTree().getId(), contentA.getId(),
619 							contentB.getId(), contentC.getId(),
620 							contentD.getId(), contentE.getId()));
621 
622 			idx = writeShallowPack(repo, 2, wants(c2), haves(c4, c5),
623 					shallows(c4));
624 			assertContent(idx, Arrays.asList(c1.getId(), c2.getId(),
625 					c1.getTree().getId(), c2.getTree().getId()));
626 		}
627 	}
628 
629 	private FileRepository setupRepoForShallowFetch() throws Exception {
630 		FileRepository repo = createBareRepository();
631 		// TestRepository will close the repo, but we need to return an open
632 		// one!
633 		repo.incrementOpen();
634 		try (TestRepository<Repository> r = new TestRepository<>(repo)) {
635 			BranchBuilder bb = r.branch("refs/heads/master");
636 			contentA = r.blob("A");
637 			contentB = r.blob("B");
638 			contentC = r.blob("C");
639 			contentD = r.blob("D");
640 			contentE = r.blob("E");
641 			c1 = bb.commit().add("a", contentA).create();
642 			c2 = bb.commit().add("b", contentB).create();
643 			c3 = bb.commit().add("c", contentC).create();
644 			c4 = bb.commit().add("d", contentD).create();
645 			c5 = bb.commit().add("e", contentE).create();
646 			r.getRevWalk().parseHeaders(c5); // fully initialize the tip RevCommit
647 			return repo;
648 		}
649 	}
650 
651 	private static PackIndex writePack(FileRepository repo,
652 			Set<? extends ObjectId> want, Set<ObjectIdSet> excludeObjects)
653 					throws IOException {
654 		try (RevWalk walk = new RevWalk(repo)) {
655 			return writePack(repo, walk, 0, want, NONE, excludeObjects);
656 		}
657 	}
658 
659 	private static PackIndex writeShallowPack(FileRepository repo, int depth,
660 			Set<? extends ObjectId> want, Set<? extends ObjectId> have,
661 			Set<? extends ObjectId> shallow) throws IOException {
662 		// During negotiation, UploadPack would have set up a DepthWalk and
663 		// marked the client's "shallow" commits. Emulate that here.
664 		try (DepthWalk.RevWalk walk = new DepthWalk.RevWalk(repo, depth - 1)) {
665 			walk.assumeShallow(shallow);
666 			return writePack(repo, walk, depth, want, have, EMPTY_ID_SET);
667 		}
668 	}
669 
670 	private static PackIndex writePack(FileRepository repo, RevWalk walk,
671 			int depth, Set<? extends ObjectId> want,
672 			Set<? extends ObjectId> have, Set<ObjectIdSet> excludeObjects)
673 					throws IOException {
674 		try (PackWriter pw = new PackWriter(repo)) {
675 			pw.setDeltaBaseAsOffset(true);
676 			pw.setReuseDeltaCommits(false);
677 			for (ObjectIdSet idx : excludeObjects) {
678 				pw.excludeObjects(idx);
679 			}
680 			if (depth > 0) {
681 				pw.setShallowPack(depth, null);
682 			}
683 			// ow doesn't need to be closed; caller closes walk.
684 			ObjectWalk ow = walk.toObjectWalkWithSameObjects();
685 
686 			pw.preparePack(NullProgressMonitor.INSTANCE, ow, want, have, NONE);
687 			File packdir = repo.getObjectDatabase().getPackDirectory();
688 			PackFile packFile = new PackFile(packdir, pw.computeName(),
689 					PackExt.PACK);
690 			try (FileOutputStream packOS = new FileOutputStream(packFile)) {
691 				pw.writePack(NullProgressMonitor.INSTANCE,
692 						NullProgressMonitor.INSTANCE, packOS);
693 			}
694 			PackFile idxFile = packFile.create(PackExt.INDEX);
695 			try (FileOutputStream idxOS = new FileOutputStream(idxFile)) {
696 				pw.writeIndex(idxOS);
697 			}
698 			return PackIndex.open(idxFile);
699 		}
700 	}
701 
702 	// TODO: testWritePackDeltasCycle()
703 	// TODO: testWritePackDeltasDepth()
704 
705 	private void writeVerifyPack1() throws IOException {
706 		final HashSet<ObjectId> interestings = new HashSet<>();
707 		interestings.add(ObjectId
708 				.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"));
709 		createVerifyOpenPack(interestings, NONE, false, false);
710 
711 		final ObjectId expectedOrder[] = new ObjectId[] {
712 				ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"),
713 				ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"),
714 				ObjectId.fromString("540a36d136cf413e4b064c2b0e0a4db60f77feab"),
715 				ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"),
716 				ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327"),
717 				ObjectId.fromString("4b825dc642cb6eb9a060e54bf8d69288fbee4904"),
718 				ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3"),
719 				ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259") };
720 
721 		assertEquals(expectedOrder.length, writer.getObjectCount());
722 		verifyObjectsOrder(expectedOrder);
723 		assertEquals("34be9032ac282b11fa9babdc2b2a93ca996c9c2f", writer
724 				.computeName().name());
725 	}
726 
727 	private void writeVerifyPack2(boolean deltaReuse) throws IOException {
728 		config.setReuseDeltas(deltaReuse);
729 		final HashSet<ObjectId> interestings = new HashSet<>();
730 		interestings.add(ObjectId
731 				.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"));
732 		final HashSet<ObjectId> uninterestings = new HashSet<>();
733 		uninterestings.add(ObjectId
734 				.fromString("540a36d136cf413e4b064c2b0e0a4db60f77feab"));
735 		createVerifyOpenPack(interestings, uninterestings, false, false);
736 
737 		final ObjectId expectedOrder[] = new ObjectId[] {
738 				ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"),
739 				ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"),
740 				ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"),
741 				ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327"),
742 				ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3") ,
743 				ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259") };
744 		if (!config.isReuseDeltas() && !config.isDeltaCompress()) {
745 			// If no deltas are in the file the final two entries swap places.
746 			swap(expectedOrder, 4, 5);
747 		}
748 		assertEquals(expectedOrder.length, writer.getObjectCount());
749 		verifyObjectsOrder(expectedOrder);
750 		assertEquals("ed3f96b8327c7c66b0f8f70056129f0769323d86", writer
751 				.computeName().name());
752 	}
753 
754 	private static void swap(ObjectId[] arr, int a, int b) {
755 		ObjectId tmp = arr[a];
756 		arr[a] = arr[b];
757 		arr[b] = tmp;
758 	}
759 
760 	private void writeVerifyPack4(final boolean thin) throws IOException {
761 		final HashSet<ObjectId> interestings = new HashSet<>();
762 		interestings.add(ObjectId
763 				.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"));
764 		final HashSet<ObjectId> uninterestings = new HashSet<>();
765 		uninterestings.add(ObjectId
766 				.fromString("c59759f143fb1fe21c197981df75a7ee00290799"));
767 		createVerifyOpenPack(interestings, uninterestings, thin, false);
768 
769 		final ObjectId writtenObjects[] = new ObjectId[] {
770 				ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"),
771 				ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"),
772 				ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259") };
773 		assertEquals(writtenObjects.length, writer.getObjectCount());
774 		ObjectId expectedObjects[];
775 		if (thin) {
776 			expectedObjects = new ObjectId[4];
777 			System.arraycopy(writtenObjects, 0, expectedObjects, 0,
778 					writtenObjects.length);
779 			expectedObjects[3] = ObjectId
780 					.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3");
781 
782 		} else {
783 			expectedObjects = writtenObjects;
784 		}
785 		verifyObjectsOrder(expectedObjects);
786 		assertEquals("cded4b74176b4456afa456768b2b5aafb41c44fc", writer
787 				.computeName().name());
788 	}
789 
790 	private void createVerifyOpenPack(final Set<ObjectId> interestings,
791 			final Set<ObjectId> uninterestings, final boolean thin,
792 			final boolean ignoreMissingUninteresting)
793 			throws MissingObjectException, IOException {
794 		createVerifyOpenPack(interestings, uninterestings, thin,
795 				ignoreMissingUninteresting, false);
796 	}
797 
798 	private void createVerifyOpenPack(final Set<ObjectId> interestings,
799 			final Set<ObjectId> uninterestings, final boolean thin,
800 			final boolean ignoreMissingUninteresting, boolean useBitmaps)
801 			throws MissingObjectException, IOException {
802 		NullProgressMonitor m = NullProgressMonitor.INSTANCE;
803 		writer = new PackWriter(config, db.newObjectReader());
804 		writer.setUseBitmaps(useBitmaps);
805 		writer.setThin(thin);
806 		writer.setIgnoreMissingUninteresting(ignoreMissingUninteresting);
807 		writer.preparePack(m, interestings, uninterestings);
808 		writer.writePack(m, m, os);
809 		writer.close();
810 		verifyOpenPack(thin);
811 	}
812 
813 	private void createVerifyOpenPack(List<RevObject> objectSource)
814 			throws MissingObjectException, IOException {
815 		NullProgressMonitor m = NullProgressMonitor.INSTANCE;
816 		writer = new PackWriter(config, db.newObjectReader());
817 		writer.preparePack(objectSource.iterator());
818 		assertEquals(objectSource.size(), writer.getObjectCount());
819 		writer.writePack(m, m, os);
820 		writer.close();
821 		verifyOpenPack(false);
822 	}
823 
824 	private void verifyOpenPack(boolean thin) throws IOException {
825 		final byte[] packData = os.toByteArray();
826 
827 		if (thin) {
828 			PackParser p = index(packData);
829 			try {
830 				p.parse(NullProgressMonitor.INSTANCE);
831 				fail("indexer should grumble about missing object");
832 			} catch (IOException x) {
833 				// expected
834 			}
835 		}
836 
837 		ObjectDirectoryPackParser p = (ObjectDirectoryPackParser) index(packData);
838 		p.setKeepEmpty(true);
839 		p.setAllowThin(thin);
840 		p.setIndexVersion(2);
841 		p.parse(NullProgressMonitor.INSTANCE);
842 		pack = p.getPack();
843 		assertNotNull("have PackFile after parsing", pack);
844 	}
845 
846 	private PackParser index(byte[] packData) throws IOException {
847 		if (inserter == null)
848 			inserter = dst.newObjectInserter();
849 		return inserter.newPackParser(new ByteArrayInputStream(packData));
850 	}
851 
852 	private void verifyObjectsOrder(ObjectId objectsOrder[]) {
853 		final List<PackIndex.MutableEntry> entries = new ArrayList<>();
854 
855 		for (MutableEntry me : pack) {
856 			entries.add(me.cloneEntry());
857 		}
858 		Collections.sort(entries, (MutableEntry o1, MutableEntry o2) -> Long
859 				.signum(o1.getOffset() - o2.getOffset()));
860 
861 		int i = 0;
862 		for (MutableEntry me : entries) {
863 			assertEquals(objectsOrder[i++].toObjectId(), me.toObjectId());
864 		}
865 	}
866 
867 	private static Set<ObjectId> haves(ObjectId... objects) {
868 		return Sets.of(objects);
869 	}
870 
871 	private static Set<ObjectId> wants(ObjectId... objects) {
872 		return Sets.of(objects);
873 	}
874 
875 	private static Set<ObjectId> shallows(ObjectId... objects) {
876 		return Sets.of(objects);
877 	}
878 }