Skip to content

Commit 802ea88

Browse files
terrellnterrelln
committed
Reduce function size in fast & dfast
Take the same approach as in PR facebook#2828 [0] to remove functions that force inline many function bodies and `switch`. Instead, create one function per "template" combination, and then switch between these functions. This allows the compiler to break the large function into many small functions, which generally helps codegen. Also, in the `extDict` modes when there is no ext-dict, call the top level function instead of the force inlined one, to save on code size. I'm specifically doing this because gcc on the parisc architecture doesn't handle the large function body well, and ends up using a lot of excess stack space. Outlining these functions fixes it.
1 parent ddae153 commit 802ea88

2 files changed

Lines changed: 71 additions & 27 deletions

File tree

‎lib/compress/zstd_double_fast.c‎

Lines changed: 35 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -468,6 +468,24 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
468468
return (size_t)(iend - anchor);
469469
}
470470

471+
#define ZSTD_GEN_DFAST_FN(dictMode, mls) \
472+
static size_t ZSTD_compressBlock_doubleFast_##dictMode##_##mls( \
473+
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \
474+
void const* src, size_t srcSize) \
475+
{ \
476+
return ZSTD_compressBlock_doubleFast_##dictMode##_generic(ms, seqStore, rep, src, srcSize, mls); \
477+
}
478+
479+
ZSTD_GEN_DFAST_FN(noDict, 4)
480+
ZSTD_GEN_DFAST_FN(noDict, 5)
481+
ZSTD_GEN_DFAST_FN(noDict, 6)
482+
ZSTD_GEN_DFAST_FN(noDict, 7)
483+
484+
ZSTD_GEN_DFAST_FN(dictMatchState, 4)
485+
ZSTD_GEN_DFAST_FN(dictMatchState, 5)
486+
ZSTD_GEN_DFAST_FN(dictMatchState, 6)
487+
ZSTD_GEN_DFAST_FN(dictMatchState, 7)
488+
471489

472490
size_t ZSTD_compressBlock_doubleFast(
473491
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
@@ -478,13 +496,13 @@ size_t ZSTD_compressBlock_doubleFast(
478496
{
479497
default: /* includes case 3 */
480498
case 4 :
481-
return ZSTD_compressBlock_doubleFast_noDict_generic(ms, seqStore, rep, src, srcSize, 4);
499+
return ZSTD_compressBlock_doubleFast_noDict_4(ms, seqStore, rep, src, srcSize);
482500
case 5 :
483-
return ZSTD_compressBlock_doubleFast_noDict_generic(ms, seqStore, rep, src, srcSize, 5);
501+
return ZSTD_compressBlock_doubleFast_noDict_5(ms, seqStore, rep, src, srcSize);
484502
case 6 :
485-
return ZSTD_compressBlock_doubleFast_noDict_generic(ms, seqStore, rep, src, srcSize, 6);
503+
return ZSTD_compressBlock_doubleFast_noDict_6(ms, seqStore, rep, src, srcSize);
486504
case 7 :
487-
return ZSTD_compressBlock_doubleFast_noDict_generic(ms, seqStore, rep, src, srcSize, 7);
505+
return ZSTD_compressBlock_doubleFast_noDict_7(ms, seqStore, rep, src, srcSize);
488506
}
489507
}
490508

@@ -498,13 +516,13 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState(
498516
{
499517
default: /* includes case 3 */
500518
case 4 :
501-
return ZSTD_compressBlock_doubleFast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 4);
519+
return ZSTD_compressBlock_doubleFast_dictMatchState_4(ms, seqStore, rep, src, srcSize);
502520
case 5 :
503-
return ZSTD_compressBlock_doubleFast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 5);
521+
return ZSTD_compressBlock_doubleFast_dictMatchState_5(ms, seqStore, rep, src, srcSize);
504522
case 6 :
505-
return ZSTD_compressBlock_doubleFast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 6);
523+
return ZSTD_compressBlock_doubleFast_dictMatchState_6(ms, seqStore, rep, src, srcSize);
506524
case 7 :
507-
return ZSTD_compressBlock_doubleFast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 7);
525+
return ZSTD_compressBlock_doubleFast_dictMatchState_7(ms, seqStore, rep, src, srcSize);
508526
}
509527
}
510528

@@ -540,7 +558,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(
540558

541559
/* if extDict is invalidated due to maxDistance, switch to "regular" variant */
542560
if (prefixStartIndex == dictStartIndex)
543-
return ZSTD_compressBlock_doubleFast_noDict_generic(ms, seqStore, rep, src, srcSize, mls);
561+
return ZSTD_compressBlock_doubleFast(ms, seqStore, rep, src, srcSize);
544562

545563
/* Search Loop */
546564
while (ip < ilimit) { /* < instead of <=, because (ip+1) */
@@ -653,6 +671,10 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(
653671
return (size_t)(iend - anchor);
654672
}
655673

674+
ZSTD_GEN_DFAST_FN(extDict, 4)
675+
ZSTD_GEN_DFAST_FN(extDict, 5)
676+
ZSTD_GEN_DFAST_FN(extDict, 6)
677+
ZSTD_GEN_DFAST_FN(extDict, 7)
656678

657679
size_t ZSTD_compressBlock_doubleFast_extDict(
658680
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
@@ -663,12 +685,12 @@ size_t ZSTD_compressBlock_doubleFast_extDict(
663685
{
664686
default: /* includes case 3 */
665687
case 4 :
666-
return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 4);
688+
return ZSTD_compressBlock_doubleFast_extDict_4(ms, seqStore, rep, src, srcSize);
667689
case 5 :
668-
return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 5);
690+
return ZSTD_compressBlock_doubleFast_extDict_5(ms, seqStore, rep, src, srcSize);
669691
case 6 :
670-
return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 6);
692+
return ZSTD_compressBlock_doubleFast_extDict_6(ms, seqStore, rep, src, srcSize);
671693
case 7 :
672-
return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 7);
694+
return ZSTD_compressBlock_doubleFast_extDict_7(ms, seqStore, rep, src, srcSize);
673695
}
674696
}

‎lib/compress/zstd_fast.c‎

Lines changed: 36 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ void ZSTD_fillHashTable(ZSTD_matchState_t* ms,
9090
* This is also the work we do at the beginning to enter the loop initially.
9191
*/
9292
FORCE_INLINE_TEMPLATE size_t
93-
ZSTD_compressBlock_fast_generic(
93+
ZSTD_compressBlock_fast_noDict_generic(
9494
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
9595
void const* src, size_t srcSize,
9696
U32 const mls)
@@ -310,6 +310,18 @@ ZSTD_compressBlock_fast_generic(
310310
goto _start;
311311
}
312312

313+
#define ZSTD_GEN_FAST_FN(dictMode, mls) \
314+
static size_t ZSTD_compressBlock_fast_##dictMode##_##mls( \
315+
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \
316+
void const* src, size_t srcSize) \
317+
{ \
318+
return ZSTD_compressBlock_fast_##dictMode##_generic(ms, seqStore, rep, src, srcSize, mls); \
319+
}
320+
321+
ZSTD_GEN_FAST_FN(noDict, 4)
322+
ZSTD_GEN_FAST_FN(noDict, 5)
323+
ZSTD_GEN_FAST_FN(noDict, 6)
324+
ZSTD_GEN_FAST_FN(noDict, 7)
313325

314326
size_t ZSTD_compressBlock_fast(
315327
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
@@ -321,13 +333,13 @@ size_t ZSTD_compressBlock_fast(
321333
{
322334
default: /* includes case 3 */
323335
case 4 :
324-
return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 4);
336+
return ZSTD_compressBlock_fast_noDict_4(ms, seqStore, rep, src, srcSize);
325337
case 5 :
326-
return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 5);
338+
return ZSTD_compressBlock_fast_noDict_5(ms, seqStore, rep, src, srcSize);
327339
case 6 :
328-
return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 6);
340+
return ZSTD_compressBlock_fast_noDict_6(ms, seqStore, rep, src, srcSize);
329341
case 7 :
330-
return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 7);
342+
return ZSTD_compressBlock_fast_noDict_7(ms, seqStore, rep, src, srcSize);
331343
}
332344
}
333345

@@ -479,6 +491,12 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic(
479491
return (size_t)(iend - anchor);
480492
}
481493

494+
495+
ZSTD_GEN_FAST_FN(dictMatchState, 4)
496+
ZSTD_GEN_FAST_FN(dictMatchState, 5)
497+
ZSTD_GEN_FAST_FN(dictMatchState, 6)
498+
ZSTD_GEN_FAST_FN(dictMatchState, 7)
499+
482500
size_t ZSTD_compressBlock_fast_dictMatchState(
483501
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
484502
void const* src, size_t srcSize)
@@ -489,13 +507,13 @@ size_t ZSTD_compressBlock_fast_dictMatchState(
489507
{
490508
default: /* includes case 3 */
491509
case 4 :
492-
return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 4);
510+
return ZSTD_compressBlock_fast_dictMatchState_4(ms, seqStore, rep, src, srcSize);
493511
case 5 :
494-
return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 5);
512+
return ZSTD_compressBlock_fast_dictMatchState_5(ms, seqStore, rep, src, srcSize);
495513
case 6 :
496-
return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 6);
514+
return ZSTD_compressBlock_fast_dictMatchState_6(ms, seqStore, rep, src, srcSize);
497515
case 7 :
498-
return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 7);
516+
return ZSTD_compressBlock_fast_dictMatchState_7(ms, seqStore, rep, src, srcSize);
499517
}
500518
}
501519

@@ -530,7 +548,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(
530548

531549
/* switch to "regular" variant if extDict is invalidated due to maxDistance */
532550
if (prefixStartIndex == dictStartIndex)
533-
return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, mls);
551+
return ZSTD_compressBlock_fast(ms, seqStore, rep, src, srcSize);
534552

535553
/* Search Loop */
536554
while (ip < ilimit) { /* < instead of <=, because (ip+1) */
@@ -603,6 +621,10 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(
603621
return (size_t)(iend - anchor);
604622
}
605623

624+
ZSTD_GEN_FAST_FN(extDict, 4)
625+
ZSTD_GEN_FAST_FN(extDict, 5)
626+
ZSTD_GEN_FAST_FN(extDict, 6)
627+
ZSTD_GEN_FAST_FN(extDict, 7)
606628

607629
size_t ZSTD_compressBlock_fast_extDict(
608630
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
@@ -613,12 +635,12 @@ size_t ZSTD_compressBlock_fast_extDict(
613635
{
614636
default: /* includes case 3 */
615637
case 4 :
616-
return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 4);
638+
return ZSTD_compressBlock_fast_extDict_4(ms, seqStore, rep, src, srcSize);
617639
case 5 :
618-
return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 5);
640+
return ZSTD_compressBlock_fast_extDict_5(ms, seqStore, rep, src, srcSize);
619641
case 6 :
620-
return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 6);
642+
return ZSTD_compressBlock_fast_extDict_6(ms, seqStore, rep, src, srcSize);
621643
case 7 :
622-
return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 7);
644+
return ZSTD_compressBlock_fast_extDict_7(ms, seqStore, rep, src, srcSize);
623645
}
624646
}

0 commit comments

Comments
 (0)