alpha/lib/ev6-stxncpy.S

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 /*
0003  * arch/alpha/lib/ev6-stxncpy.S
0004  * 21264 version contributed by Rick Gorton <rick.gorton@api-networks.com>
0005  *
0006  * Copy no more than COUNT bytes of the null-terminated string from
0007  * SRC to DST.
0008  *
0009  * This is an internal routine used by strncpy, stpncpy, and strncat.
0010  * As such, it uses special linkage conventions to make implementation
0011  * of these public functions more efficient.
0012  *
0013  * On input:
0014  *  t9 = return address
0015  *  a0 = DST
0016  *  a1 = SRC
0017  *  a2 = COUNT
0018  *
0019  * Furthermore, COUNT may not be zero.
0020  *
0021  * On output:
0022  *  t0  = last word written
0023  *  t10 = bitmask (with one bit set) indicating the byte position of
0024  *        the end of the range specified by COUNT
0025  *  t12 = bitmask (with one bit set) indicating the last byte written
0026  *  a0  = unaligned address of the last *word* written
0027  *  a2  = the number of full words left in COUNT
0028  *
0029  * Furthermore, v0, a3-a5, t11, and $at are untouched.
0030  *
0031  * Much of the information about 21264 scheduling/coding comes from:
0032  *  Compiler Writer's Guide for the Alpha 21264
0033  *  abbreviated as 'CWG' in other comments here
0034  *  ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html
0035  * Scheduling notation:
0036  *  E   - either cluster
0037  *  U   - upper subcluster; U0 - subcluster U0; U1 - subcluster U1
0038  *  L   - lower subcluster; L0 - subcluster L0; L1 - subcluster L1
0039  * Try not to change the actual algorithm if possible for consistency.
0040  */
0041
0042 #include <asm/regdef.h>
0043
0044     .set noat
0045     .set noreorder
0046
0047     .text
0048
0049 /* There is a problem with either gdb (as of 4.16) or gas (as of 2.7) that
0050    doesn't like putting the entry point for a procedure somewhere in the
0051    middle of the procedure descriptor.  Work around this by putting the
0052    aligned copy in its own procedure descriptor */
0053
0054
0055     .ent stxncpy_aligned
0056     .align 4
0057 stxncpy_aligned:
0058     .frame sp, 0, t9, 0
0059     .prologue 0
0060
0061     /* On entry to this basic block:
0062        t0 == the first destination word for masking back in
0063        t1 == the first source word.  */
0064
0065     /* Create the 1st output word and detect 0's in the 1st input word.  */
0066     lda t2, -1      # E : build a mask against false zero
0067     mskqh   t2, a1, t2  # U :   detection in the src word (stall)
0068     mskqh   t1, a1, t3  # U :
0069     ornot   t1, t2, t2  # E : (stall)
0070
0071     mskql   t0, a1, t0  # U : assemble the first output word
0072     cmpbge  zero, t2, t8    # E : bits set iff null found
0073     or  t0, t3, t0  # E : (stall)
0074     beq a2, $a_eoc  # U :
0075
0076     bne t8, $a_eos  # U :
0077     nop
0078     nop
0079     nop
0080
0081     /* On entry to this basic block:
0082        t0 == a source word not containing a null.  */
0083
0084     /*
0085      * nops here to:
0086      *  separate store quads from load quads
0087      *  limit of 1 bcond/quad to permit training
0088      */
0089 $a_loop:
0090     stq_u   t0, 0(a0)   # L :
0091     addq    a0, 8, a0   # E :
0092     subq    a2, 1, a2   # E :
0093     nop
0094
0095     ldq_u   t0, 0(a1)   # L :
0096     addq    a1, 8, a1   # E :
0097     cmpbge  zero, t0, t8    # E :
0098     beq a2, $a_eoc      # U :
0099
0100     beq t8, $a_loop # U :
0101     nop
0102     nop
0103     nop
0104
0105     /* Take care of the final (partial) word store.  At this point
0106        the end-of-count bit is set in t8 iff it applies.
0107
0108        On entry to this basic block we have:
0109        t0 == the source word containing the null
0110        t8 == the cmpbge mask that found it.  */
0111
0112 $a_eos:
0113     negq    t8, t12     # E : find low bit set
0114     and t8, t12, t12    # E : (stall)
0115     /* For the sake of the cache, don't read a destination word
0116        if we're not going to need it.  */
0117     and t12, 0x80, t6   # E : (stall)
0118     bne t6, 1f      # U : (stall)
0119
0120     /* We're doing a partial word store and so need to combine
0121        our source and original destination words.  */
0122     ldq_u   t1, 0(a0)   # L :
0123     subq    t12, 1, t6  # E :
0124     or  t12, t6, t8 # E : (stall)
0125     zapnot  t0, t8, t0  # U : clear src bytes > null (stall)
0126
0127     zap t1, t8, t1  # .. e1 : clear dst bytes <= null
0128     or  t0, t1, t0  # e1    : (stall)
0129     nop
0130     nop
0131
0132 1:  stq_u   t0, 0(a0)   # L :
0133     ret (t9)        # L0 : Latency=3
0134     nop
0135     nop
0136
0137     /* Add the end-of-count bit to the eos detection bitmask.  */
0138 $a_eoc:
0139     or  t10, t8, t8 # E :
0140     br  $a_eos      # L0 : Latency=3
0141     nop
0142     nop
0143
0144     .end stxncpy_aligned
0145
0146     .align 4
0147     .ent __stxncpy
0148     .globl __stxncpy
0149 __stxncpy:
0150     .frame sp, 0, t9, 0
0151     .prologue 0
0152
0153     /* Are source and destination co-aligned?  */
0154     xor a0, a1, t1  # E :
0155     and a0, 7, t0   # E : find dest misalignment
0156     and t1, 7, t1   # E : (stall)
0157     addq    a2, t0, a2  # E : bias count by dest misalignment (stall)
0158
0159     subq    a2, 1, a2   # E :
0160     and a2, 7, t2   # E : (stall)
0161     srl a2, 3, a2   # U : a2 = loop counter = (count - 1)/8 (stall)
0162     addq    zero, 1, t10    # E :
0163
0164     sll t10, t2, t10    # U : t10 = bitmask of last count byte
0165     bne t1, $unaligned  # U :
0166     /* We are co-aligned; take care of a partial first word.  */
0167     ldq_u   t1, 0(a1)   # L : load first src word
0168     addq    a1, 8, a1   # E :
0169
0170     beq t0, stxncpy_aligned     # U : avoid loading dest word if not needed
0171     ldq_u   t0, 0(a0)   # L :
0172     nop
0173     nop
0174
0175     br  stxncpy_aligned # .. e1 :
0176     nop
0177     nop
0178     nop
0179
0180
0181
0182 /* The source and destination are not co-aligned.  Align the destination
0183    and cope.  We have to be very careful about not reading too much and
0184    causing a SEGV.  */
0185
0186     .align 4
0187 $u_head:
0188     /* We know just enough now to be able to assemble the first
0189        full source word.  We can still find a zero at the end of it
0190        that prevents us from outputting the whole thing.
0191
0192        On entry to this basic block:
0193        t0 == the first dest word, unmasked
0194        t1 == the shifted low bits of the first source word
0195        t6 == bytemask that is -1 in dest word bytes */
0196
0197     ldq_u   t2, 8(a1)   # L : Latency=3 load second src word
0198     addq    a1, 8, a1   # E :
0199     mskql   t0, a0, t0  # U : mask trailing garbage in dst
0200     extqh   t2, a1, t4  # U : (3 cycle stall on t2)
0201
0202     or  t1, t4, t1  # E : first aligned src word complete (stall)
0203     mskqh   t1, a0, t1  # U : mask leading garbage in src (stall)
0204     or  t0, t1, t0  # E : first output word complete (stall)
0205     or  t0, t6, t6  # E : mask original data for zero test (stall)
0206
0207     cmpbge  zero, t6, t8    # E :
0208     beq a2, $u_eocfin   # U :
0209     lda t6, -1      # E :
0210     nop
0211
0212     bne t8, $u_final    # U :
0213     mskql   t6, a1, t6  # U : mask out bits already seen
0214     stq_u   t0, 0(a0)   # L : store first output word
0215     or      t6, t2, t2  # E : (stall)
0216
0217     cmpbge  zero, t2, t8    # E : find nulls in second partial
0218     addq    a0, 8, a0   # E :
0219     subq    a2, 1, a2   # E :
0220     bne t8, $u_late_head_exit   # U :
0221
0222     /* Finally, we've got all the stupid leading edge cases taken care
0223        of and we can set up to enter the main loop.  */
0224     extql   t2, a1, t1  # U : position hi-bits of lo word
0225     beq a2, $u_eoc  # U :
0226     ldq_u   t2, 8(a1)   # L : read next high-order source word
0227     addq    a1, 8, a1   # E :
0228
0229     extqh   t2, a1, t0  # U : position lo-bits of hi word (stall)
0230     cmpbge  zero, t2, t8    # E :
0231     nop
0232     bne t8, $u_eos  # U :
0233
0234     /* Unaligned copy main loop.  In order to avoid reading too much,
0235        the loop is structured to detect zeros in aligned source words.
0236        This has, unfortunately, effectively pulled half of a loop
0237        iteration out into the head and half into the tail, but it does
0238        prevent nastiness from accumulating in the very thing we want
0239        to run as fast as possible.
0240
0241        On entry to this basic block:
0242        t0 == the shifted low-order bits from the current source word
0243        t1 == the shifted high-order bits from the previous source word
0244        t2 == the unshifted current source word
0245
0246        We further know that t2 does not contain a null terminator.  */
0247
0248     .align 4
0249 $u_loop:
0250     or  t0, t1, t0  # E : current dst word now complete
0251     subq    a2, 1, a2   # E : decrement word count
0252     extql   t2, a1, t1  # U : extract low bits for next time
0253     addq    a0, 8, a0   # E :
0254
0255     stq_u   t0, -8(a0)  # U : save the current word
0256     beq a2, $u_eoc  # U :
0257     ldq_u   t2, 8(a1)   # U : Latency=3 load high word for next time
0258     addq    a1, 8, a1   # E :
0259
0260     extqh   t2, a1, t0  # U : extract low bits (2 cycle stall)
0261     cmpbge  zero, t2, t8    # E : test new word for eos
0262     nop
0263     beq t8, $u_loop # U :
0264
0265     /* We've found a zero somewhere in the source word we just read.
0266        If it resides in the lower half, we have one (probably partial)
0267        word to write out, and if it resides in the upper half, we
0268        have one full and one partial word left to write out.
0269
0270        On entry to this basic block:
0271        t0 == the shifted low-order bits from the current source word
0272        t1 == the shifted high-order bits from the previous source word
0273        t2 == the unshifted current source word.  */
0274 $u_eos:
0275     or  t0, t1, t0  # E : first (partial) source word complete
0276     nop
0277     cmpbge  zero, t0, t8    # E : is the null in this first bit? (stall)
0278     bne t8, $u_final    # U : (stall)
0279
0280     stq_u   t0, 0(a0)   # L : the null was in the high-order bits
0281     addq    a0, 8, a0   # E :
0282     subq    a2, 1, a2   # E :
0283     nop
0284
0285 $u_late_head_exit:
0286     extql   t2, a1, t0  # U :
0287     cmpbge  zero, t0, t8    # E :
0288     or  t8, t10, t6 # E : (stall)
0289     cmoveq  a2, t6, t8  # E : Latency=2, extra map slot (stall)
0290
0291     /* Take care of a final (probably partial) result word.
0292        On entry to this basic block:
0293        t0 == assembled source word
0294        t8 == cmpbge mask that found the null.  */
0295 $u_final:
0296     negq    t8, t6      # E : isolate low bit set
0297     and t6, t8, t12 # E : (stall)
0298     and t12, 0x80, t6   # E : avoid dest word load if we can (stall)
0299     bne t6, 1f      # U : (stall)
0300
0301     ldq_u   t1, 0(a0)   # L :
0302     subq    t12, 1, t6  # E :
0303     or  t6, t12, t8 # E : (stall)
0304     zapnot  t0, t8, t0  # U : kill source bytes > null
0305
0306     zap t1, t8, t1  # U : kill dest bytes <= null
0307     or  t0, t1, t0  # E : (stall)
0308     nop
0309     nop
0310
0311 1:  stq_u   t0, 0(a0)   # L :
0312     ret (t9)        # L0 : Latency=3
0313
0314       /* Got to end-of-count before end of string.
0315          On entry to this basic block:
0316          t1 == the shifted high-order bits from the previous source word  */
0317 $u_eoc:
0318     and a1, 7, t6   # E : avoid final load if possible
0319     sll t10, t6, t6 # U : (stall)
0320     and t6, 0xff, t6    # E : (stall)
0321     bne t6, 1f      # U : (stall)
0322
0323     ldq_u   t2, 8(a1)   # L : load final src word
0324     nop
0325     extqh   t2, a1, t0  # U : extract low bits for last word (stall)
0326     or  t1, t0, t1  # E : (stall)
0327
0328 1:  cmpbge  zero, t1, t8    # E :
0329     mov t1, t0      # E :
0330
0331 $u_eocfin:          # end-of-count, final word
0332     or  t10, t8, t8 # E :
0333     br  $u_final    # L0 : Latency=3
0334
0335     /* Unaligned copy entry point.  */
0336     .align 4
0337 $unaligned:
0338
0339     ldq_u   t1, 0(a1)   # L : load first source word
0340     and a0, 7, t4   # E : find dest misalignment
0341     and a1, 7, t5   # E : find src misalignment
0342     /* Conditionally load the first destination word and a bytemask
0343        with 0xff indicating that the destination byte is sacrosanct.  */
0344     mov zero, t0    # E :
0345
0346     mov zero, t6    # E :
0347     beq t4, 1f      # U :
0348     ldq_u   t0, 0(a0)   # L :
0349     lda t6, -1      # E :
0350
0351     mskql   t6, a0, t6  # U :
0352     nop
0353     nop
0354     subq    a1, t4, a1  # E : sub dest misalignment from src addr
0355
0356     /* If source misalignment is larger than dest misalignment, we need
0357        extra startup checks to avoid SEGV.  */
0358
0359 1:  cmplt   t4, t5, t12 # E :
0360     extql   t1, a1, t1  # U : shift src into place
0361     lda t2, -1      # E : for creating masks later
0362     beq t12, $u_head    # U : (stall)
0363
0364     extql   t2, a1, t2  # U :
0365     cmpbge  zero, t1, t8    # E : is there a zero?
0366     andnot  t2, t6, t2  # E : dest mask for a single word copy
0367     or  t8, t10, t5 # E : test for end-of-count too
0368
0369     cmpbge  zero, t2, t3    # E :
0370     cmoveq  a2, t5, t8  # E : Latency=2, extra map slot
0371     nop         # E : keep with cmoveq
0372     andnot  t8, t3, t8  # E : (stall)
0373
0374     beq t8, $u_head # U :
0375     /* At this point we've found a zero in the first partial word of
0376        the source.  We need to isolate the valid source data and mask
0377        it into the original destination data.  (Incidentally, we know
0378        that we'll need at least one byte of that original dest word.) */
0379     ldq_u   t0, 0(a0)   # L :
0380     negq    t8, t6      # E : build bitmask of bytes <= zero
0381     mskqh   t1, t4, t1  # U :
0382
0383     and t6, t8, t12 # E :
0384     subq    t12, 1, t6  # E : (stall)
0385     or  t6, t12, t8 # E : (stall)
0386     zapnot  t2, t8, t2  # U : prepare source word; mirror changes (stall)
0387
0388     zapnot  t1, t8, t1  # U : to source validity mask
0389     andnot  t0, t2, t0  # E : zero place for source to reside
0390     or  t0, t1, t0  # E : and put it there (stall both t0, t1)
0391     stq_u   t0, 0(a0)   # L : (stall)
0392
0393     ret (t9)        # L0 : Latency=3
0394     nop
0395     nop
0396     nop
0397
0398     .end __stxncpy