FFmpeg
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
rgb2rgb_template.c
Go to the documentation of this file.
1 /*
2  * software RGB to RGB converter
3  * pluralize by software PAL8 to RGB converter
4  * software YUV to YUV converter
5  * software YUV to RGB converter
6  * Written by Nick Kurshev.
7  * palette & YUV & runtime CPU stuff by Michael (michaelni@gmx.at)
8  * lot of big-endian byte order fixes by Alex Beregszaszi
9  *
10  * This file is part of FFmpeg.
11  *
12  * FFmpeg is free software; you can redistribute it and/or
13  * modify it under the terms of the GNU Lesser General Public
14  * License as published by the Free Software Foundation; either
15  * version 2.1 of the License, or (at your option) any later version.
16  *
17  * FFmpeg is distributed in the hope that it will be useful,
18  * but WITHOUT ANY WARRANTY; without even the implied warranty of
19  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20  * Lesser General Public License for more details.
21  *
22  * You should have received a copy of the GNU Lesser General Public
23  * License along with FFmpeg; if not, write to the Free Software
24  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
25  */
26 
27 #include <stddef.h>
28 
29 #include "libavutil/attributes.h"
30 
31 #undef PREFETCH
32 #undef MOVNTQ
33 #undef EMMS
34 #undef SFENCE
35 #undef PAVGB
36 
37 #if COMPILE_TEMPLATE_AMD3DNOW
38 #define PREFETCH "prefetch"
39 #define PAVGB "pavgusb"
40 #elif COMPILE_TEMPLATE_MMXEXT
41 #define PREFETCH "prefetchnta"
42 #define PAVGB "pavgb"
43 #else
44 #define PREFETCH " # nop"
45 #endif
46 
47 #if COMPILE_TEMPLATE_AMD3DNOW
48 /* On K6 femms is faster than emms. On K7 femms is directly mapped to emms. */
49 #define EMMS "femms"
50 #else
51 #define EMMS "emms"
52 #endif
53 
54 #if COMPILE_TEMPLATE_MMXEXT
55 #define MOVNTQ "movntq"
56 #define SFENCE "sfence"
57 #else
58 #define MOVNTQ "movq"
59 #define SFENCE " # nop"
60 #endif
61 
62 #if !COMPILE_TEMPLATE_SSE2
63 
64 #if !COMPILE_TEMPLATE_AMD3DNOW
65 
66 static inline void RENAME(rgb24tobgr32)(const uint8_t *src, uint8_t *dst, int src_size)
67 {
68  uint8_t *dest = dst;
69  const uint8_t *s = src;
70  const uint8_t *end;
71  const uint8_t *mm_end;
72  end = s + src_size;
73  __asm__ volatile(PREFETCH" %0"::"m"(*s):"memory");
74  mm_end = end - 23;
75  __asm__ volatile("movq %0, %%mm7"::"m"(mask32a):"memory");
76  while (s < mm_end) {
77  __asm__ volatile(
78  PREFETCH" 32(%1) \n\t"
79  "movd (%1), %%mm0 \n\t"
80  "punpckldq 3(%1), %%mm0 \n\t"
81  "movd 6(%1), %%mm1 \n\t"
82  "punpckldq 9(%1), %%mm1 \n\t"
83  "movd 12(%1), %%mm2 \n\t"
84  "punpckldq 15(%1), %%mm2 \n\t"
85  "movd 18(%1), %%mm3 \n\t"
86  "punpckldq 21(%1), %%mm3 \n\t"
87  "por %%mm7, %%mm0 \n\t"
88  "por %%mm7, %%mm1 \n\t"
89  "por %%mm7, %%mm2 \n\t"
90  "por %%mm7, %%mm3 \n\t"
91  MOVNTQ" %%mm0, (%0) \n\t"
92  MOVNTQ" %%mm1, 8(%0) \n\t"
93  MOVNTQ" %%mm2, 16(%0) \n\t"
94  MOVNTQ" %%mm3, 24(%0)"
95  :: "r"(dest), "r"(s)
96  :"memory");
97  dest += 32;
98  s += 24;
99  }
100  __asm__ volatile(SFENCE:::"memory");
101  __asm__ volatile(EMMS:::"memory");
102  while (s < end) {
103  *dest++ = *s++;
104  *dest++ = *s++;
105  *dest++ = *s++;
106  *dest++ = 255;
107  }
108 }
109 
110 #define STORE_BGR24_MMX \
111  "psrlq $8, %%mm2 \n\t" \
112  "psrlq $8, %%mm3 \n\t" \
113  "psrlq $8, %%mm6 \n\t" \
114  "psrlq $8, %%mm7 \n\t" \
115  "pand "MANGLE(mask24l)", %%mm0\n\t" \
116  "pand "MANGLE(mask24l)", %%mm1\n\t" \
117  "pand "MANGLE(mask24l)", %%mm4\n\t" \
118  "pand "MANGLE(mask24l)", %%mm5\n\t" \
119  "pand "MANGLE(mask24h)", %%mm2\n\t" \
120  "pand "MANGLE(mask24h)", %%mm3\n\t" \
121  "pand "MANGLE(mask24h)", %%mm6\n\t" \
122  "pand "MANGLE(mask24h)", %%mm7\n\t" \
123  "por %%mm2, %%mm0 \n\t" \
124  "por %%mm3, %%mm1 \n\t" \
125  "por %%mm6, %%mm4 \n\t" \
126  "por %%mm7, %%mm5 \n\t" \
127  \
128  "movq %%mm1, %%mm2 \n\t" \
129  "movq %%mm4, %%mm3 \n\t" \
130  "psllq $48, %%mm2 \n\t" \
131  "psllq $32, %%mm3 \n\t" \
132  "por %%mm2, %%mm0 \n\t" \
133  "psrlq $16, %%mm1 \n\t" \
134  "psrlq $32, %%mm4 \n\t" \
135  "psllq $16, %%mm5 \n\t" \
136  "por %%mm3, %%mm1 \n\t" \
137  "por %%mm5, %%mm4 \n\t" \
138  \
139  MOVNTQ" %%mm0, (%0) \n\t" \
140  MOVNTQ" %%mm1, 8(%0) \n\t" \
141  MOVNTQ" %%mm4, 16(%0)"
142 
143 
144 static inline void RENAME(rgb32tobgr24)(const uint8_t *src, uint8_t *dst, int src_size)
145 {
146  uint8_t *dest = dst;
147  const uint8_t *s = src;
148  const uint8_t *end;
149  const uint8_t *mm_end;
150  end = s + src_size;
151  __asm__ volatile(PREFETCH" %0"::"m"(*s):"memory");
152  mm_end = end - 31;
153  while (s < mm_end) {
154  __asm__ volatile(
155  PREFETCH" 32(%1) \n\t"
156  "movq (%1), %%mm0 \n\t"
157  "movq 8(%1), %%mm1 \n\t"
158  "movq 16(%1), %%mm4 \n\t"
159  "movq 24(%1), %%mm5 \n\t"
160  "movq %%mm0, %%mm2 \n\t"
161  "movq %%mm1, %%mm3 \n\t"
162  "movq %%mm4, %%mm6 \n\t"
163  "movq %%mm5, %%mm7 \n\t"
164  STORE_BGR24_MMX
165  :: "r"(dest), "r"(s)
166  :"memory");
167  dest += 24;
168  s += 32;
169  }
170  __asm__ volatile(SFENCE:::"memory");
171  __asm__ volatile(EMMS:::"memory");
172  while (s < end) {
173  *dest++ = *s++;
174  *dest++ = *s++;
175  *dest++ = *s++;
176  s++;
177  }
178 }
179 
180 /*
181  original by Strepto/Astral
182  ported to gcc & bugfixed: A'rpi
183  MMXEXT, 3DNOW optimization by Nick Kurshev
184  32-bit C version, and and&add trick by Michael Niedermayer
185 */
186 static inline void RENAME(rgb15to16)(const uint8_t *src, uint8_t *dst, int src_size)
187 {
188  register const uint8_t* s=src;
189  register uint8_t* d=dst;
190  register const uint8_t *end;
191  const uint8_t *mm_end;
192  end = s + src_size;
193  __asm__ volatile(PREFETCH" %0"::"m"(*s));
194  __asm__ volatile("movq %0, %%mm4"::"m"(mask15s));
195  mm_end = end - 15;
196  while (s<mm_end) {
197  __asm__ volatile(
198  PREFETCH" 32(%1) \n\t"
199  "movq (%1), %%mm0 \n\t"
200  "movq 8(%1), %%mm2 \n\t"
201  "movq %%mm0, %%mm1 \n\t"
202  "movq %%mm2, %%mm3 \n\t"
203  "pand %%mm4, %%mm0 \n\t"
204  "pand %%mm4, %%mm2 \n\t"
205  "paddw %%mm1, %%mm0 \n\t"
206  "paddw %%mm3, %%mm2 \n\t"
207  MOVNTQ" %%mm0, (%0) \n\t"
208  MOVNTQ" %%mm2, 8(%0)"
209  :: "r"(d), "r"(s)
210  );
211  d+=16;
212  s+=16;
213  }
214  __asm__ volatile(SFENCE:::"memory");
215  __asm__ volatile(EMMS:::"memory");
216  mm_end = end - 3;
217  while (s < mm_end) {
218  register unsigned x= *((const uint32_t *)s);
219  *((uint32_t *)d) = (x&0x7FFF7FFF) + (x&0x7FE07FE0);
220  d+=4;
221  s+=4;
222  }
223  if (s < end) {
224  register unsigned short x= *((const uint16_t *)s);
225  *((uint16_t *)d) = (x&0x7FFF) + (x&0x7FE0);
226  }
227 }
228 
229 static inline void RENAME(rgb16to15)(const uint8_t *src, uint8_t *dst, int src_size)
230 {
231  register const uint8_t* s=src;
232  register uint8_t* d=dst;
233  register const uint8_t *end;
234  const uint8_t *mm_end;
235  end = s + src_size;
236  __asm__ volatile(PREFETCH" %0"::"m"(*s));
237  __asm__ volatile("movq %0, %%mm7"::"m"(mask15rg));
238  __asm__ volatile("movq %0, %%mm6"::"m"(mask15b));
239  mm_end = end - 15;
240  while (s<mm_end) {
241  __asm__ volatile(
242  PREFETCH" 32(%1) \n\t"
243  "movq (%1), %%mm0 \n\t"
244  "movq 8(%1), %%mm2 \n\t"
245  "movq %%mm0, %%mm1 \n\t"
246  "movq %%mm2, %%mm3 \n\t"
247  "psrlq $1, %%mm0 \n\t"
248  "psrlq $1, %%mm2 \n\t"
249  "pand %%mm7, %%mm0 \n\t"
250  "pand %%mm7, %%mm2 \n\t"
251  "pand %%mm6, %%mm1 \n\t"
252  "pand %%mm6, %%mm3 \n\t"
253  "por %%mm1, %%mm0 \n\t"
254  "por %%mm3, %%mm2 \n\t"
255  MOVNTQ" %%mm0, (%0) \n\t"
256  MOVNTQ" %%mm2, 8(%0)"
257  :: "r"(d), "r"(s)
258  );
259  d+=16;
260  s+=16;
261  }
262  __asm__ volatile(SFENCE:::"memory");
263  __asm__ volatile(EMMS:::"memory");
264  mm_end = end - 3;
265  while (s < mm_end) {
266  register uint32_t x= *((const uint32_t*)s);
267  *((uint32_t *)d) = ((x>>1)&0x7FE07FE0) | (x&0x001F001F);
268  s+=4;
269  d+=4;
270  }
271  if (s < end) {
272  register uint16_t x= *((const uint16_t*)s);
273  *((uint16_t *)d) = ((x>>1)&0x7FE0) | (x&0x001F);
274  }
275 }
276 
277 static inline void RENAME(rgb32to16)(const uint8_t *src, uint8_t *dst, int src_size)
278 {
279  const uint8_t *s = src;
280  const uint8_t *end;
281  const uint8_t *mm_end;
282  uint16_t *d = (uint16_t *)dst;
283  end = s + src_size;
284  mm_end = end - 15;
285  __asm__ volatile(
286  "movq %3, %%mm5 \n\t"
287  "movq %4, %%mm6 \n\t"
288  "movq %5, %%mm7 \n\t"
289  "jmp 2f \n\t"
290  ".p2align 4 \n\t"
291  "1: \n\t"
292  PREFETCH" 32(%1) \n\t"
293  "movd (%1), %%mm0 \n\t"
294  "movd 4(%1), %%mm3 \n\t"
295  "punpckldq 8(%1), %%mm0 \n\t"
296  "punpckldq 12(%1), %%mm3 \n\t"
297  "movq %%mm0, %%mm1 \n\t"
298  "movq %%mm3, %%mm4 \n\t"
299  "pand %%mm6, %%mm0 \n\t"
300  "pand %%mm6, %%mm3 \n\t"
301  "pmaddwd %%mm7, %%mm0 \n\t"
302  "pmaddwd %%mm7, %%mm3 \n\t"
303  "pand %%mm5, %%mm1 \n\t"
304  "pand %%mm5, %%mm4 \n\t"
305  "por %%mm1, %%mm0 \n\t"
306  "por %%mm4, %%mm3 \n\t"
307  "psrld $5, %%mm0 \n\t"
308  "pslld $11, %%mm3 \n\t"
309  "por %%mm3, %%mm0 \n\t"
310  MOVNTQ" %%mm0, (%0) \n\t"
311  "add $16, %1 \n\t"
312  "add $8, %0 \n\t"
313  "2: \n\t"
314  "cmp %2, %1 \n\t"
315  " jb 1b \n\t"
316  : "+r" (d), "+r"(s)
317  : "r" (mm_end), "m" (mask3216g), "m" (mask3216br), "m" (mul3216)
318  );
319  __asm__ volatile(SFENCE:::"memory");
320  __asm__ volatile(EMMS:::"memory");
321  while (s < end) {
322  register int rgb = *(const uint32_t*)s; s += 4;
323  *d++ = ((rgb&0xFF)>>3) + ((rgb&0xFC00)>>5) + ((rgb&0xF80000)>>8);
324  }
325 }
326 
327 static inline void RENAME(rgb32tobgr16)(const uint8_t *src, uint8_t *dst, int src_size)
328 {
329  const uint8_t *s = src;
330  const uint8_t *end;
331  const uint8_t *mm_end;
332  uint16_t *d = (uint16_t *)dst;
333  end = s + src_size;
334  __asm__ volatile(PREFETCH" %0"::"m"(*src):"memory");
335  __asm__ volatile(
336  "movq %0, %%mm7 \n\t"
337  "movq %1, %%mm6 \n\t"
338  ::"m"(red_16mask),"m"(green_16mask));
339  mm_end = end - 15;
340  while (s < mm_end) {
341  __asm__ volatile(
342  PREFETCH" 32(%1) \n\t"
343  "movd (%1), %%mm0 \n\t"
344  "movd 4(%1), %%mm3 \n\t"
345  "punpckldq 8(%1), %%mm0 \n\t"
346  "punpckldq 12(%1), %%mm3 \n\t"
347  "movq %%mm0, %%mm1 \n\t"
348  "movq %%mm0, %%mm2 \n\t"
349  "movq %%mm3, %%mm4 \n\t"
350  "movq %%mm3, %%mm5 \n\t"
351  "psllq $8, %%mm0 \n\t"
352  "psllq $8, %%mm3 \n\t"
353  "pand %%mm7, %%mm0 \n\t"
354  "pand %%mm7, %%mm3 \n\t"
355  "psrlq $5, %%mm1 \n\t"
356  "psrlq $5, %%mm4 \n\t"
357  "pand %%mm6, %%mm1 \n\t"
358  "pand %%mm6, %%mm4 \n\t"
359  "psrlq $19, %%mm2 \n\t"
360  "psrlq $19, %%mm5 \n\t"
361  "pand %2, %%mm2 \n\t"
362  "pand %2, %%mm5 \n\t"
363  "por %%mm1, %%mm0 \n\t"
364  "por %%mm4, %%mm3 \n\t"
365  "por %%mm2, %%mm0 \n\t"
366  "por %%mm5, %%mm3 \n\t"
367  "psllq $16, %%mm3 \n\t"
368  "por %%mm3, %%mm0 \n\t"
369  MOVNTQ" %%mm0, (%0) \n\t"
370  :: "r"(d),"r"(s),"m"(blue_16mask):"memory");
371  d += 4;
372  s += 16;
373  }
374  __asm__ volatile(SFENCE:::"memory");
375  __asm__ volatile(EMMS:::"memory");
376  while (s < end) {
377  register int rgb = *(const uint32_t*)s; s += 4;
378  *d++ = ((rgb&0xF8)<<8) + ((rgb&0xFC00)>>5) + ((rgb&0xF80000)>>19);
379  }
380 }
381 
382 static inline void RENAME(rgb32to15)(const uint8_t *src, uint8_t *dst, int src_size)
383 {
384  const uint8_t *s = src;
385  const uint8_t *end;
386  const uint8_t *mm_end;
387  uint16_t *d = (uint16_t *)dst;
388  end = s + src_size;
389  mm_end = end - 15;
390  __asm__ volatile(
391  "movq %3, %%mm5 \n\t"
392  "movq %4, %%mm6 \n\t"
393  "movq %5, %%mm7 \n\t"
394  "jmp 2f \n\t"
395  ".p2align 4 \n\t"
396  "1: \n\t"
397  PREFETCH" 32(%1) \n\t"
398  "movd (%1), %%mm0 \n\t"
399  "movd 4(%1), %%mm3 \n\t"
400  "punpckldq 8(%1), %%mm0 \n\t"
401  "punpckldq 12(%1), %%mm3 \n\t"
402  "movq %%mm0, %%mm1 \n\t"
403  "movq %%mm3, %%mm4 \n\t"
404  "pand %%mm6, %%mm0 \n\t"
405  "pand %%mm6, %%mm3 \n\t"
406  "pmaddwd %%mm7, %%mm0 \n\t"
407  "pmaddwd %%mm7, %%mm3 \n\t"
408  "pand %%mm5, %%mm1 \n\t"
409  "pand %%mm5, %%mm4 \n\t"
410  "por %%mm1, %%mm0 \n\t"
411  "por %%mm4, %%mm3 \n\t"
412  "psrld $6, %%mm0 \n\t"
413  "pslld $10, %%mm3 \n\t"
414  "por %%mm3, %%mm0 \n\t"
415  MOVNTQ" %%mm0, (%0) \n\t"
416  "add $16, %1 \n\t"
417  "add $8, %0 \n\t"
418  "2: \n\t"
419  "cmp %2, %1 \n\t"
420  " jb 1b \n\t"
421  : "+r" (d), "+r"(s)
422  : "r" (mm_end), "m" (mask3215g), "m" (mask3216br), "m" (mul3215)
423  );
424  __asm__ volatile(SFENCE:::"memory");
425  __asm__ volatile(EMMS:::"memory");
426  while (s < end) {
427  register int rgb = *(const uint32_t*)s; s += 4;
428  *d++ = ((rgb&0xFF)>>3) + ((rgb&0xF800)>>6) + ((rgb&0xF80000)>>9);
429  }
430 }
431 
432 static inline void RENAME(rgb32tobgr15)(const uint8_t *src, uint8_t *dst, int src_size)
433 {
434  const uint8_t *s = src;
435  const uint8_t *end;
436  const uint8_t *mm_end;
437  uint16_t *d = (uint16_t *)dst;
438  end = s + src_size;
439  __asm__ volatile(PREFETCH" %0"::"m"(*src):"memory");
440  __asm__ volatile(
441  "movq %0, %%mm7 \n\t"
442  "movq %1, %%mm6 \n\t"
443  ::"m"(red_15mask),"m"(green_15mask));
444  mm_end = end - 15;
445  while (s < mm_end) {
446  __asm__ volatile(
447  PREFETCH" 32(%1) \n\t"
448  "movd (%1), %%mm0 \n\t"
449  "movd 4(%1), %%mm3 \n\t"
450  "punpckldq 8(%1), %%mm0 \n\t"
451  "punpckldq 12(%1), %%mm3 \n\t"
452  "movq %%mm0, %%mm1 \n\t"
453  "movq %%mm0, %%mm2 \n\t"
454  "movq %%mm3, %%mm4 \n\t"
455  "movq %%mm3, %%mm5 \n\t"
456  "psllq $7, %%mm0 \n\t"
457  "psllq $7, %%mm3 \n\t"
458  "pand %%mm7, %%mm0 \n\t"
459  "pand %%mm7, %%mm3 \n\t"
460  "psrlq $6, %%mm1 \n\t"
461  "psrlq $6, %%mm4 \n\t"
462  "pand %%mm6, %%mm1 \n\t"
463  "pand %%mm6, %%mm4 \n\t"
464  "psrlq $19, %%mm2 \n\t"
465  "psrlq $19, %%mm5 \n\t"
466  "pand %2, %%mm2 \n\t"
467  "pand %2, %%mm5 \n\t"
468  "por %%mm1, %%mm0 \n\t"
469  "por %%mm4, %%mm3 \n\t"
470  "por %%mm2, %%mm0 \n\t"
471  "por %%mm5, %%mm3 \n\t"
472  "psllq $16, %%mm3 \n\t"
473  "por %%mm3, %%mm0 \n\t"
474  MOVNTQ" %%mm0, (%0) \n\t"
475  ::"r"(d),"r"(s),"m"(blue_15mask):"memory");
476  d += 4;
477  s += 16;
478  }
479  __asm__ volatile(SFENCE:::"memory");
480  __asm__ volatile(EMMS:::"memory");
481  while (s < end) {
482  register int rgb = *(const uint32_t*)s; s += 4;
483  *d++ = ((rgb&0xF8)<<7) + ((rgb&0xF800)>>6) + ((rgb&0xF80000)>>19);
484  }
485 }
486 
487 static inline void RENAME(rgb24tobgr16)(const uint8_t *src, uint8_t *dst, int src_size)
488 {
489  const uint8_t *s = src;
490  const uint8_t *end;
491  const uint8_t *mm_end;
492  uint16_t *d = (uint16_t *)dst;
493  end = s + src_size;
494  __asm__ volatile(PREFETCH" %0"::"m"(*src):"memory");
495  __asm__ volatile(
496  "movq %0, %%mm7 \n\t"
497  "movq %1, %%mm6 \n\t"
498  ::"m"(red_16mask),"m"(green_16mask));
499  mm_end = end - 11;
500  while (s < mm_end) {
501  __asm__ volatile(
502  PREFETCH" 32(%1) \n\t"
503  "movd (%1), %%mm0 \n\t"
504  "movd 3(%1), %%mm3 \n\t"
505  "punpckldq 6(%1), %%mm0 \n\t"
506  "punpckldq 9(%1), %%mm3 \n\t"
507  "movq %%mm0, %%mm1 \n\t"
508  "movq %%mm0, %%mm2 \n\t"
509  "movq %%mm3, %%mm4 \n\t"
510  "movq %%mm3, %%mm5 \n\t"
511  "psrlq $3, %%mm0 \n\t"
512  "psrlq $3, %%mm3 \n\t"
513  "pand %2, %%mm0 \n\t"
514  "pand %2, %%mm3 \n\t"
515  "psrlq $5, %%mm1 \n\t"
516  "psrlq $5, %%mm4 \n\t"
517  "pand %%mm6, %%mm1 \n\t"
518  "pand %%mm6, %%mm4 \n\t"
519  "psrlq $8, %%mm2 \n\t"
520  "psrlq $8, %%mm5 \n\t"
521  "pand %%mm7, %%mm2 \n\t"
522  "pand %%mm7, %%mm5 \n\t"
523  "por %%mm1, %%mm0 \n\t"
524  "por %%mm4, %%mm3 \n\t"
525  "por %%mm2, %%mm0 \n\t"
526  "por %%mm5, %%mm3 \n\t"
527  "psllq $16, %%mm3 \n\t"
528  "por %%mm3, %%mm0 \n\t"
529  MOVNTQ" %%mm0, (%0) \n\t"
530  ::"r"(d),"r"(s),"m"(blue_16mask):"memory");
531  d += 4;
532  s += 12;
533  }
534  __asm__ volatile(SFENCE:::"memory");
535  __asm__ volatile(EMMS:::"memory");
536  while (s < end) {
537  const int b = *s++;
538  const int g = *s++;
539  const int r = *s++;
540  *d++ = (b>>3) | ((g&0xFC)<<3) | ((r&0xF8)<<8);
541  }
542 }
543 
544 static inline void RENAME(rgb24to16)(const uint8_t *src, uint8_t *dst, int src_size)
545 {
546  const uint8_t *s = src;
547  const uint8_t *end;
548  const uint8_t *mm_end;
549  uint16_t *d = (uint16_t *)dst;
550  end = s + src_size;
551  __asm__ volatile(PREFETCH" %0"::"m"(*src):"memory");
552  __asm__ volatile(
553  "movq %0, %%mm7 \n\t"
554  "movq %1, %%mm6 \n\t"
555  ::"m"(red_16mask),"m"(green_16mask));
556  mm_end = end - 15;
557  while (s < mm_end) {
558  __asm__ volatile(
559  PREFETCH" 32(%1) \n\t"
560  "movd (%1), %%mm0 \n\t"
561  "movd 3(%1), %%mm3 \n\t"
562  "punpckldq 6(%1), %%mm0 \n\t"
563  "punpckldq 9(%1), %%mm3 \n\t"
564  "movq %%mm0, %%mm1 \n\t"
565  "movq %%mm0, %%mm2 \n\t"
566  "movq %%mm3, %%mm4 \n\t"
567  "movq %%mm3, %%mm5 \n\t"
568  "psllq $8, %%mm0 \n\t"
569  "psllq $8, %%mm3 \n\t"
570  "pand %%mm7, %%mm0 \n\t"
571  "pand %%mm7, %%mm3 \n\t"
572  "psrlq $5, %%mm1 \n\t"
573  "psrlq $5, %%mm4 \n\t"
574  "pand %%mm6, %%mm1 \n\t"
575  "pand %%mm6, %%mm4 \n\t"
576  "psrlq $19, %%mm2 \n\t"
577  "psrlq $19, %%mm5 \n\t"
578  "pand %2, %%mm2 \n\t"
579  "pand %2, %%mm5 \n\t"
580  "por %%mm1, %%mm0 \n\t"
581  "por %%mm4, %%mm3 \n\t"
582  "por %%mm2, %%mm0 \n\t"
583  "por %%mm5, %%mm3 \n\t"
584  "psllq $16, %%mm3 \n\t"
585  "por %%mm3, %%mm0 \n\t"
586  MOVNTQ" %%mm0, (%0) \n\t"
587  ::"r"(d),"r"(s),"m"(blue_16mask):"memory");
588  d += 4;
589  s += 12;
590  }
591  __asm__ volatile(SFENCE:::"memory");
592  __asm__ volatile(EMMS:::"memory");
593  while (s < end) {
594  const int r = *s++;
595  const int g = *s++;
596  const int b = *s++;
597  *d++ = (b>>3) | ((g&0xFC)<<3) | ((r&0xF8)<<8);
598  }
599 }
600 
601 static inline void RENAME(rgb24tobgr15)(const uint8_t *src, uint8_t *dst, int src_size)
602 {
603  const uint8_t *s = src;
604  const uint8_t *end;
605  const uint8_t *mm_end;
606  uint16_t *d = (uint16_t *)dst;
607  end = s + src_size;
608  __asm__ volatile(PREFETCH" %0"::"m"(*src):"memory");
609  __asm__ volatile(
610  "movq %0, %%mm7 \n\t"
611  "movq %1, %%mm6 \n\t"
612  ::"m"(red_15mask),"m"(green_15mask));
613  mm_end = end - 11;
614  while (s < mm_end) {
615  __asm__ volatile(
616  PREFETCH" 32(%1) \n\t"
617  "movd (%1), %%mm0 \n\t"
618  "movd 3(%1), %%mm3 \n\t"
619  "punpckldq 6(%1), %%mm0 \n\t"
620  "punpckldq 9(%1), %%mm3 \n\t"
621  "movq %%mm0, %%mm1 \n\t"
622  "movq %%mm0, %%mm2 \n\t"
623  "movq %%mm3, %%mm4 \n\t"
624  "movq %%mm3, %%mm5 \n\t"
625  "psrlq $3, %%mm0 \n\t"
626  "psrlq $3, %%mm3 \n\t"
627  "pand %2, %%mm0 \n\t"
628  "pand %2, %%mm3 \n\t"
629  "psrlq $6, %%mm1 \n\t"
630  "psrlq $6, %%mm4 \n\t"
631  "pand %%mm6, %%mm1 \n\t"
632  "pand %%mm6, %%mm4 \n\t"
633  "psrlq $9, %%mm2 \n\t"
634  "psrlq $9, %%mm5 \n\t"
635  "pand %%mm7, %%mm2 \n\t"
636  "pand %%mm7, %%mm5 \n\t"
637  "por %%mm1, %%mm0 \n\t"
638  "por %%mm4, %%mm3 \n\t"
639  "por %%mm2, %%mm0 \n\t"
640  "por %%mm5, %%mm3 \n\t"
641  "psllq $16, %%mm3 \n\t"
642  "por %%mm3, %%mm0 \n\t"
643  MOVNTQ" %%mm0, (%0) \n\t"
644  ::"r"(d),"r"(s),"m"(blue_15mask):"memory");
645  d += 4;
646  s += 12;
647  }
648  __asm__ volatile(SFENCE:::"memory");
649  __asm__ volatile(EMMS:::"memory");
650  while (s < end) {
651  const int b = *s++;
652  const int g = *s++;
653  const int r = *s++;
654  *d++ = (b>>3) | ((g&0xF8)<<2) | ((r&0xF8)<<7);
655  }
656 }
657 
658 static inline void RENAME(rgb24to15)(const uint8_t *src, uint8_t *dst, int src_size)
659 {
660  const uint8_t *s = src;
661  const uint8_t *end;
662  const uint8_t *mm_end;
663  uint16_t *d = (uint16_t *)dst;
664  end = s + src_size;
665  __asm__ volatile(PREFETCH" %0"::"m"(*src):"memory");
666  __asm__ volatile(
667  "movq %0, %%mm7 \n\t"
668  "movq %1, %%mm6 \n\t"
669  ::"m"(red_15mask),"m"(green_15mask));
670  mm_end = end - 15;
671  while (s < mm_end) {
672  __asm__ volatile(
673  PREFETCH" 32(%1) \n\t"
674  "movd (%1), %%mm0 \n\t"
675  "movd 3(%1), %%mm3 \n\t"
676  "punpckldq 6(%1), %%mm0 \n\t"
677  "punpckldq 9(%1), %%mm3 \n\t"
678  "movq %%mm0, %%mm1 \n\t"
679  "movq %%mm0, %%mm2 \n\t"
680  "movq %%mm3, %%mm4 \n\t"
681  "movq %%mm3, %%mm5 \n\t"
682  "psllq $7, %%mm0 \n\t"
683  "psllq $7, %%mm3 \n\t"
684  "pand %%mm7, %%mm0 \n\t"
685  "pand %%mm7, %%mm3 \n\t"
686  "psrlq $6, %%mm1 \n\t"
687  "psrlq $6, %%mm4 \n\t"
688  "pand %%mm6, %%mm1 \n\t"
689  "pand %%mm6, %%mm4 \n\t"
690  "psrlq $19, %%mm2 \n\t"
691  "psrlq $19, %%mm5 \n\t"
692  "pand %2, %%mm2 \n\t"
693  "pand %2, %%mm5 \n\t"
694  "por %%mm1, %%mm0 \n\t"
695  "por %%mm4, %%mm3 \n\t"
696  "por %%mm2, %%mm0 \n\t"
697  "por %%mm5, %%mm3 \n\t"
698  "psllq $16, %%mm3 \n\t"
699  "por %%mm3, %%mm0 \n\t"
700  MOVNTQ" %%mm0, (%0) \n\t"
701  ::"r"(d),"r"(s),"m"(blue_15mask):"memory");
702  d += 4;
703  s += 12;
704  }
705  __asm__ volatile(SFENCE:::"memory");
706  __asm__ volatile(EMMS:::"memory");
707  while (s < end) {
708  const int r = *s++;
709  const int g = *s++;
710  const int b = *s++;
711  *d++ = (b>>3) | ((g&0xF8)<<2) | ((r&0xF8)<<7);
712  }
713 }
714 
715 static inline void RENAME(rgb15tobgr24)(const uint8_t *src, uint8_t *dst, int src_size)
716 {
717  const uint16_t *end;
718  const uint16_t *mm_end;
719  uint8_t *d = dst;
720  const uint16_t *s = (const uint16_t*)src;
721  end = s + src_size/2;
722  __asm__ volatile(PREFETCH" %0"::"m"(*s):"memory");
723  mm_end = end - 7;
724  while (s < mm_end) {
725  __asm__ volatile(
726  PREFETCH" 32(%1) \n\t"
727  "movq (%1), %%mm0 \n\t"
728  "movq (%1), %%mm1 \n\t"
729  "movq (%1), %%mm2 \n\t"
730  "pand %2, %%mm0 \n\t"
731  "pand %3, %%mm1 \n\t"
732  "pand %4, %%mm2 \n\t"
733  "psllq $5, %%mm0 \n\t"
734  "pmulhw "MANGLE(mul15_mid)", %%mm0 \n\t"
735  "pmulhw "MANGLE(mul15_mid)", %%mm1 \n\t"
736  "pmulhw "MANGLE(mul15_hi)", %%mm2 \n\t"
737  "movq %%mm0, %%mm3 \n\t"
738  "movq %%mm1, %%mm4 \n\t"
739  "movq %%mm2, %%mm5 \n\t"
740  "punpcklwd %5, %%mm0 \n\t"
741  "punpcklwd %5, %%mm1 \n\t"
742  "punpcklwd %5, %%mm2 \n\t"
743  "punpckhwd %5, %%mm3 \n\t"
744  "punpckhwd %5, %%mm4 \n\t"
745  "punpckhwd %5, %%mm5 \n\t"
746  "psllq $8, %%mm1 \n\t"
747  "psllq $16, %%mm2 \n\t"
748  "por %%mm1, %%mm0 \n\t"
749  "por %%mm2, %%mm0 \n\t"
750  "psllq $8, %%mm4 \n\t"
751  "psllq $16, %%mm5 \n\t"
752  "por %%mm4, %%mm3 \n\t"
753  "por %%mm5, %%mm3 \n\t"
754 
755  "movq %%mm0, %%mm6 \n\t"
756  "movq %%mm3, %%mm7 \n\t"
757 
758  "movq 8(%1), %%mm0 \n\t"
759  "movq 8(%1), %%mm1 \n\t"
760  "movq 8(%1), %%mm2 \n\t"
761  "pand %2, %%mm0 \n\t"
762  "pand %3, %%mm1 \n\t"
763  "pand %4, %%mm2 \n\t"
764  "psllq $5, %%mm0 \n\t"
765  "pmulhw "MANGLE(mul15_mid)", %%mm0 \n\t"
766  "pmulhw "MANGLE(mul15_mid)", %%mm1 \n\t"
767  "pmulhw "MANGLE(mul15_hi)", %%mm2 \n\t"
768  "movq %%mm0, %%mm3 \n\t"
769  "movq %%mm1, %%mm4 \n\t"
770  "movq %%mm2, %%mm5 \n\t"
771  "punpcklwd %5, %%mm0 \n\t"
772  "punpcklwd %5, %%mm1 \n\t"
773  "punpcklwd %5, %%mm2 \n\t"
774  "punpckhwd %5, %%mm3 \n\t"
775  "punpckhwd %5, %%mm4 \n\t"
776  "punpckhwd %5, %%mm5 \n\t"
777  "psllq $8, %%mm1 \n\t"
778  "psllq $16, %%mm2 \n\t"
779  "por %%mm1, %%mm0 \n\t"
780  "por %%mm2, %%mm0 \n\t"
781  "psllq $8, %%mm4 \n\t"
782  "psllq $16, %%mm5 \n\t"
783  "por %%mm4, %%mm3 \n\t"
784  "por %%mm5, %%mm3 \n\t"
785 
786  :"=m"(*d)
787  :"r"(s),"m"(mask15b),"m"(mask15g),"m"(mask15r), "m"(mmx_null)
788  :"memory");
789  /* borrowed 32 to 24 */
790  __asm__ volatile(
791  "movq %%mm0, %%mm4 \n\t"
792  "movq %%mm3, %%mm5 \n\t"
793  "movq %%mm6, %%mm0 \n\t"
794  "movq %%mm7, %%mm1 \n\t"
795 
796  "movq %%mm4, %%mm6 \n\t"
797  "movq %%mm5, %%mm7 \n\t"
798  "movq %%mm0, %%mm2 \n\t"
799  "movq %%mm1, %%mm3 \n\t"
800 
801  STORE_BGR24_MMX
802 
803  :: "r"(d), "m"(*s)
804  :"memory");
805  d += 24;
806  s += 8;
807  }
808  __asm__ volatile(SFENCE:::"memory");
809  __asm__ volatile(EMMS:::"memory");
810  while (s < end) {
811  register uint16_t bgr;
812  bgr = *s++;
813  *d++ = ((bgr&0x1F)<<3) | ((bgr&0x1F)>>2);
814  *d++ = ((bgr&0x3E0)>>2) | ((bgr&0x3E0)>>7);
815  *d++ = ((bgr&0x7C00)>>7) | ((bgr&0x7C00)>>12);
816  }
817 }
818 
819 static inline void RENAME(rgb16tobgr24)(const uint8_t *src, uint8_t *dst, int src_size)
820 {
821  const uint16_t *end;
822  const uint16_t *mm_end;
823  uint8_t *d = (uint8_t *)dst;
824  const uint16_t *s = (const uint16_t *)src;
825  end = s + src_size/2;
826  __asm__ volatile(PREFETCH" %0"::"m"(*s):"memory");
827  mm_end = end - 7;
828  while (s < mm_end) {
829  __asm__ volatile(
830  PREFETCH" 32(%1) \n\t"
831  "movq (%1), %%mm0 \n\t"
832  "movq (%1), %%mm1 \n\t"
833  "movq (%1), %%mm2 \n\t"
834  "pand %2, %%mm0 \n\t"
835  "pand %3, %%mm1 \n\t"
836  "pand %4, %%mm2 \n\t"
837  "psllq $5, %%mm0 \n\t"
838  "psrlq $1, %%mm2 \n\t"
839  "pmulhw "MANGLE(mul15_mid)", %%mm0 \n\t"
840  "pmulhw "MANGLE(mul16_mid)", %%mm1 \n\t"
841  "pmulhw "MANGLE(mul15_hi)", %%mm2 \n\t"
842  "movq %%mm0, %%mm3 \n\t"
843  "movq %%mm1, %%mm4 \n\t"
844  "movq %%mm2, %%mm5 \n\t"
845  "punpcklwd %5, %%mm0 \n\t"
846  "punpcklwd %5, %%mm1 \n\t"
847  "punpcklwd %5, %%mm2 \n\t"
848  "punpckhwd %5, %%mm3 \n\t"
849  "punpckhwd %5, %%mm4 \n\t"
850  "punpckhwd %5, %%mm5 \n\t"
851  "psllq $8, %%mm1 \n\t"
852  "psllq $16, %%mm2 \n\t"
853  "por %%mm1, %%mm0 \n\t"
854  "por %%mm2, %%mm0 \n\t"
855  "psllq $8, %%mm4 \n\t"
856  "psllq $16, %%mm5 \n\t"
857  "por %%mm4, %%mm3 \n\t"
858  "por %%mm5, %%mm3 \n\t"
859 
860  "movq %%mm0, %%mm6 \n\t"
861  "movq %%mm3, %%mm7 \n\t"
862 
863  "movq 8(%1), %%mm0 \n\t"
864  "movq 8(%1), %%mm1 \n\t"
865  "movq 8(%1), %%mm2 \n\t"
866  "pand %2, %%mm0 \n\t"
867  "pand %3, %%mm1 \n\t"
868  "pand %4, %%mm2 \n\t"
869  "psllq $5, %%mm0 \n\t"
870  "psrlq $1, %%mm2 \n\t"
871  "pmulhw "MANGLE(mul15_mid)", %%mm0 \n\t"
872  "pmulhw "MANGLE(mul16_mid)", %%mm1 \n\t"
873  "pmulhw "MANGLE(mul15_hi)", %%mm2 \n\t"
874  "movq %%mm0, %%mm3 \n\t"
875  "movq %%mm1, %%mm4 \n\t"
876  "movq %%mm2, %%mm5 \n\t"
877  "punpcklwd %5, %%mm0 \n\t"
878  "punpcklwd %5, %%mm1 \n\t"
879  "punpcklwd %5, %%mm2 \n\t"
880  "punpckhwd %5, %%mm3 \n\t"
881  "punpckhwd %5, %%mm4 \n\t"
882  "punpckhwd %5, %%mm5 \n\t"
883  "psllq $8, %%mm1 \n\t"
884  "psllq $16, %%mm2 \n\t"
885  "por %%mm1, %%mm0 \n\t"
886  "por %%mm2, %%mm0 \n\t"
887  "psllq $8, %%mm4 \n\t"
888  "psllq $16, %%mm5 \n\t"
889  "por %%mm4, %%mm3 \n\t"
890  "por %%mm5, %%mm3 \n\t"
891  :"=m"(*d)
892  :"r"(s),"m"(mask16b),"m"(mask16g),"m"(mask16r),"m"(mmx_null)
893  :"memory");
894  /* borrowed 32 to 24 */
895  __asm__ volatile(
896  "movq %%mm0, %%mm4 \n\t"
897  "movq %%mm3, %%mm5 \n\t"
898  "movq %%mm6, %%mm0 \n\t"
899  "movq %%mm7, %%mm1 \n\t"
900 
901  "movq %%mm4, %%mm6 \n\t"
902  "movq %%mm5, %%mm7 \n\t"
903  "movq %%mm0, %%mm2 \n\t"
904  "movq %%mm1, %%mm3 \n\t"
905 
906  STORE_BGR24_MMX
907 
908  :: "r"(d), "m"(*s)
909  :"memory");
910  d += 24;
911  s += 8;
912  }
913  __asm__ volatile(SFENCE:::"memory");
914  __asm__ volatile(EMMS:::"memory");
915  while (s < end) {
916  register uint16_t bgr;
917  bgr = *s++;
918  *d++ = ((bgr&0x1F)<<3) | ((bgr&0x1F)>>2);
919  *d++ = ((bgr&0x7E0)>>3) | ((bgr&0x7E0)>>9);
920  *d++ = ((bgr&0xF800)>>8) | ((bgr&0xF800)>>13);
921  }
922 }
923 
924 /*
925  * mm0 = 00 B3 00 B2 00 B1 00 B0
926  * mm1 = 00 G3 00 G2 00 G1 00 G0
927  * mm2 = 00 R3 00 R2 00 R1 00 R0
928  * mm6 = FF FF FF FF FF FF FF FF
929  * mm7 = 00 00 00 00 00 00 00 00
930  */
931 #define PACK_RGB32 \
932  "packuswb %%mm7, %%mm0 \n\t" /* 00 00 00 00 B3 B2 B1 B0 */ \
933  "packuswb %%mm7, %%mm1 \n\t" /* 00 00 00 00 G3 G2 G1 G0 */ \
934  "packuswb %%mm7, %%mm2 \n\t" /* 00 00 00 00 R3 R2 R1 R0 */ \
935  "punpcklbw %%mm1, %%mm0 \n\t" /* G3 B3 G2 B2 G1 B1 G0 B0 */ \
936  "punpcklbw %%mm6, %%mm2 \n\t" /* FF R3 FF R2 FF R1 FF R0 */ \
937  "movq %%mm0, %%mm3 \n\t" \
938  "punpcklwd %%mm2, %%mm0 \n\t" /* FF R1 G1 B1 FF R0 G0 B0 */ \
939  "punpckhwd %%mm2, %%mm3 \n\t" /* FF R3 G3 B3 FF R2 G2 B2 */ \
940  MOVNTQ" %%mm0, (%0) \n\t" \
941  MOVNTQ" %%mm3, 8(%0) \n\t" \
942 
943 static inline void RENAME(rgb15to32)(const uint8_t *src, uint8_t *dst, int src_size)
944 {
945  const uint16_t *end;
946  const uint16_t *mm_end;
947  uint8_t *d = dst;
948  const uint16_t *s = (const uint16_t *)src;
949  end = s + src_size/2;
950  __asm__ volatile(PREFETCH" %0"::"m"(*s):"memory");
951  __asm__ volatile("pxor %%mm7,%%mm7 \n\t":::"memory");
952  __asm__ volatile("pcmpeqd %%mm6,%%mm6 \n\t":::"memory");
953  mm_end = end - 3;
954  while (s < mm_end) {
955  __asm__ volatile(
956  PREFETCH" 32(%1) \n\t"
957  "movq (%1), %%mm0 \n\t"
958  "movq (%1), %%mm1 \n\t"
959  "movq (%1), %%mm2 \n\t"
960  "pand %2, %%mm0 \n\t"
961  "pand %3, %%mm1 \n\t"
962  "pand %4, %%mm2 \n\t"
963  "psllq $5, %%mm0 \n\t"
964  "pmulhw %5, %%mm0 \n\t"
965  "pmulhw %5, %%mm1 \n\t"
966  "pmulhw "MANGLE(mul15_hi)", %%mm2 \n\t"
967  PACK_RGB32
968  ::"r"(d),"r"(s),"m"(mask15b),"m"(mask15g),"m"(mask15r) ,"m"(mul15_mid)
969  :"memory");
970  d += 16;
971  s += 4;
972  }
973  __asm__ volatile(SFENCE:::"memory");
974  __asm__ volatile(EMMS:::"memory");
975  while (s < end) {
976  register uint16_t bgr;
977  bgr = *s++;
978  *d++ = ((bgr&0x1F)<<3) | ((bgr&0x1F)>>2);
979  *d++ = ((bgr&0x3E0)>>2) | ((bgr&0x3E0)>>7);
980  *d++ = ((bgr&0x7C00)>>7) | ((bgr&0x7C00)>>12);
981  *d++ = 255;
982  }
983 }
984 
985 static inline void RENAME(rgb16to32)(const uint8_t *src, uint8_t *dst, int src_size)
986 {
987  const uint16_t *end;
988  const uint16_t *mm_end;
989  uint8_t *d = dst;
990  const uint16_t *s = (const uint16_t*)src;
991  end = s + src_size/2;
992  __asm__ volatile(PREFETCH" %0"::"m"(*s):"memory");
993  __asm__ volatile("pxor %%mm7,%%mm7 \n\t":::"memory");
994  __asm__ volatile("pcmpeqd %%mm6,%%mm6 \n\t":::"memory");
995  mm_end = end - 3;
996  while (s < mm_end) {
997  __asm__ volatile(
998  PREFETCH" 32(%1) \n\t"
999  "movq (%1), %%mm0 \n\t"
1000  "movq (%1), %%mm1 \n\t"
1001  "movq (%1), %%mm2 \n\t"
1002  "pand %2, %%mm0 \n\t"
1003  "pand %3, %%mm1 \n\t"
1004  "pand %4, %%mm2 \n\t"
1005  "psllq $5, %%mm0 \n\t"
1006  "psrlq $1, %%mm2 \n\t"
1007  "pmulhw %5, %%mm0 \n\t"
1008  "pmulhw "MANGLE(mul16_mid)", %%mm1 \n\t"
1009  "pmulhw "MANGLE(mul15_hi)", %%mm2 \n\t"
1010  PACK_RGB32
1011  ::"r"(d),"r"(s),"m"(mask16b),"m"(mask16g),"m"(mask16r),"m"(mul15_mid)
1012  :"memory");
1013  d += 16;
1014  s += 4;
1015  }
1016  __asm__ volatile(SFENCE:::"memory");
1017  __asm__ volatile(EMMS:::"memory");
1018  while (s < end) {
1019  register uint16_t bgr;
1020  bgr = *s++;
1021  *d++ = ((bgr&0x1F)<<3) | ((bgr&0x1F)>>2);
1022  *d++ = ((bgr&0x7E0)>>3) | ((bgr&0x7E0)>>9);
1023  *d++ = ((bgr&0xF800)>>8) | ((bgr&0xF800)>>13);
1024  *d++ = 255;
1025  }
1026 }
1027 
1028 static inline void RENAME(shuffle_bytes_2103)(const uint8_t *src, uint8_t *dst, int src_size)
1029 {
1030  x86_reg idx = 15 - src_size;
1031  const uint8_t *s = src-idx;
1032  uint8_t *d = dst-idx;
1033  __asm__ volatile(
1034  "test %0, %0 \n\t"
1035  "jns 2f \n\t"
1036  PREFETCH" (%1, %0) \n\t"
1037  "movq %3, %%mm7 \n\t"
1038  "pxor %4, %%mm7 \n\t"
1039  "movq %%mm7, %%mm6 \n\t"
1040  "pxor %5, %%mm7 \n\t"
1041  ".p2align 4 \n\t"
1042  "1: \n\t"
1043  PREFETCH" 32(%1, %0) \n\t"
1044  "movq (%1, %0), %%mm0 \n\t"
1045  "movq 8(%1, %0), %%mm1 \n\t"
1046 # if COMPILE_TEMPLATE_MMXEXT
1047  "pshufw $177, %%mm0, %%mm3 \n\t"
1048  "pshufw $177, %%mm1, %%mm5 \n\t"
1049  "pand %%mm7, %%mm0 \n\t"
1050  "pand %%mm6, %%mm3 \n\t"
1051  "pand %%mm7, %%mm1 \n\t"
1052  "pand %%mm6, %%mm5 \n\t"
1053  "por %%mm3, %%mm0 \n\t"
1054  "por %%mm5, %%mm1 \n\t"
1055 # else
1056  "movq %%mm0, %%mm2 \n\t"
1057  "movq %%mm1, %%mm4 \n\t"
1058  "pand %%mm7, %%mm0 \n\t"
1059  "pand %%mm6, %%mm2 \n\t"
1060  "pand %%mm7, %%mm1 \n\t"
1061  "pand %%mm6, %%mm4 \n\t"
1062  "movq %%mm2, %%mm3 \n\t"
1063  "movq %%mm4, %%mm5 \n\t"
1064  "pslld $16, %%mm2 \n\t"
1065  "psrld $16, %%mm3 \n\t"
1066  "pslld $16, %%mm4 \n\t"
1067  "psrld $16, %%mm5 \n\t"
1068  "por %%mm2, %%mm0 \n\t"
1069  "por %%mm4, %%mm1 \n\t"
1070  "por %%mm3, %%mm0 \n\t"
1071  "por %%mm5, %%mm1 \n\t"
1072 # endif
1073  MOVNTQ" %%mm0, (%2, %0) \n\t"
1074  MOVNTQ" %%mm1, 8(%2, %0) \n\t"
1075  "add $16, %0 \n\t"
1076  "js 1b \n\t"
1077  SFENCE" \n\t"
1078  EMMS" \n\t"
1079  "2: \n\t"
1080  : "+&r"(idx)
1081  : "r" (s), "r" (d), "m" (mask32b), "m" (mask32r), "m" (mmx_one)
1082  : "memory");
1083  for (; idx<15; idx+=4) {
1084  register int v = *(const uint32_t *)&s[idx], g = v & 0xff00ff00;
1085  v &= 0xff00ff;
1086  *(uint32_t *)&d[idx] = (v>>16) + g + (v<<16);
1087  }
1088 }
1089 
1090 static inline void RENAME(rgb24tobgr24)(const uint8_t *src, uint8_t *dst, int src_size)
1091 {
1092  unsigned i;
1093  x86_reg mmx_size= 23 - src_size;
1094  __asm__ volatile (
1095  "test %%"REG_a", %%"REG_a" \n\t"
1096  "jns 2f \n\t"
1097  "movq "MANGLE(mask24r)", %%mm5 \n\t"
1098  "movq "MANGLE(mask24g)", %%mm6 \n\t"
1099  "movq "MANGLE(mask24b)", %%mm7 \n\t"
1100  ".p2align 4 \n\t"
1101  "1: \n\t"
1102  PREFETCH" 32(%1, %%"REG_a") \n\t"
1103  "movq (%1, %%"REG_a"), %%mm0 \n\t" // BGR BGR BG
1104  "movq (%1, %%"REG_a"), %%mm1 \n\t" // BGR BGR BG
1105  "movq 2(%1, %%"REG_a"), %%mm2 \n\t" // R BGR BGR B
1106  "psllq $16, %%mm0 \n\t" // 00 BGR BGR
1107  "pand %%mm5, %%mm0 \n\t"
1108  "pand %%mm6, %%mm1 \n\t"
1109  "pand %%mm7, %%mm2 \n\t"
1110  "por %%mm0, %%mm1 \n\t"
1111  "por %%mm2, %%mm1 \n\t"
1112  "movq 6(%1, %%"REG_a"), %%mm0 \n\t" // BGR BGR BG
1113  MOVNTQ" %%mm1, (%2, %%"REG_a") \n\t" // RGB RGB RG
1114  "movq 8(%1, %%"REG_a"), %%mm1 \n\t" // R BGR BGR B
1115  "movq 10(%1, %%"REG_a"), %%mm2 \n\t" // GR BGR BGR
1116  "pand %%mm7, %%mm0 \n\t"
1117  "pand %%mm5, %%mm1 \n\t"
1118  "pand %%mm6, %%mm2 \n\t"
1119  "por %%mm0, %%mm1 \n\t"
1120  "por %%mm2, %%mm1 \n\t"
1121  "movq 14(%1, %%"REG_a"), %%mm0 \n\t" // R BGR BGR B
1122  MOVNTQ" %%mm1, 8(%2, %%"REG_a") \n\t" // B RGB RGB R
1123  "movq 16(%1, %%"REG_a"), %%mm1 \n\t" // GR BGR BGR
1124  "movq 18(%1, %%"REG_a"), %%mm2 \n\t" // BGR BGR BG
1125  "pand %%mm6, %%mm0 \n\t"
1126  "pand %%mm7, %%mm1 \n\t"
1127  "pand %%mm5, %%mm2 \n\t"
1128  "por %%mm0, %%mm1 \n\t"
1129  "por %%mm2, %%mm1 \n\t"
1130  MOVNTQ" %%mm1, 16(%2, %%"REG_a") \n\t"
1131  "add $24, %%"REG_a" \n\t"
1132  " js 1b \n\t"
1133  "2: \n\t"
1134  : "+a" (mmx_size)
1135  : "r" (src-mmx_size), "r"(dst-mmx_size)
1136  );
1137 
1138  __asm__ volatile(SFENCE:::"memory");
1139  __asm__ volatile(EMMS:::"memory");
1140 
1141  if (mmx_size==23) return; //finished, was multiple of 8
1142 
1143  src+= src_size;
1144  dst+= src_size;
1145  src_size= 23-mmx_size;
1146  src-= src_size;
1147  dst-= src_size;
1148  for (i=0; i<src_size; i+=3) {
1149  register uint8_t x;
1150  x = src[i + 2];
1151  dst[i + 1] = src[i + 1];
1152  dst[i + 2] = src[i + 0];
1153  dst[i + 0] = x;
1154  }
1155 }
1156 
1157 static inline void RENAME(yuvPlanartoyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1158  int width, int height,
1159  int lumStride, int chromStride, int dstStride, int vertLumPerChroma)
1160 {
1161  int y;
1162  const x86_reg chromWidth= width>>1;
1163  for (y=0; y<height; y++) {
1164  //FIXME handle 2 lines at once (fewer prefetches, reuse some chroma, but very likely memory-limited anyway)
1165  __asm__ volatile(
1166  "xor %%"REG_a", %%"REG_a" \n\t"
1167  ".p2align 4 \n\t"
1168  "1: \n\t"
1169  PREFETCH" 32(%1, %%"REG_a", 2) \n\t"
1170  PREFETCH" 32(%2, %%"REG_a") \n\t"
1171  PREFETCH" 32(%3, %%"REG_a") \n\t"
1172  "movq (%2, %%"REG_a"), %%mm0 \n\t" // U(0)
1173  "movq %%mm0, %%mm2 \n\t" // U(0)
1174  "movq (%3, %%"REG_a"), %%mm1 \n\t" // V(0)
1175  "punpcklbw %%mm1, %%mm0 \n\t" // UVUV UVUV(0)
1176  "punpckhbw %%mm1, %%mm2 \n\t" // UVUV UVUV(8)
1177 
1178  "movq (%1, %%"REG_a",2), %%mm3 \n\t" // Y(0)
1179  "movq 8(%1, %%"REG_a",2), %%mm5 \n\t" // Y(8)
1180  "movq %%mm3, %%mm4 \n\t" // Y(0)
1181  "movq %%mm5, %%mm6 \n\t" // Y(8)
1182  "punpcklbw %%mm0, %%mm3 \n\t" // YUYV YUYV(0)
1183  "punpckhbw %%mm0, %%mm4 \n\t" // YUYV YUYV(4)
1184  "punpcklbw %%mm2, %%mm5 \n\t" // YUYV YUYV(8)
1185  "punpckhbw %%mm2, %%mm6 \n\t" // YUYV YUYV(12)
1186 
1187  MOVNTQ" %%mm3, (%0, %%"REG_a", 4) \n\t"
1188  MOVNTQ" %%mm4, 8(%0, %%"REG_a", 4) \n\t"
1189  MOVNTQ" %%mm5, 16(%0, %%"REG_a", 4) \n\t"
1190  MOVNTQ" %%mm6, 24(%0, %%"REG_a", 4) \n\t"
1191 
1192  "add $8, %%"REG_a" \n\t"
1193  "cmp %4, %%"REG_a" \n\t"
1194  " jb 1b \n\t"
1195  ::"r"(dst), "r"(ysrc), "r"(usrc), "r"(vsrc), "g" (chromWidth)
1196  : "%"REG_a
1197  );
1198  if ((y&(vertLumPerChroma-1)) == vertLumPerChroma-1) {
1199  usrc += chromStride;
1200  vsrc += chromStride;
1201  }
1202  ysrc += lumStride;
1203  dst += dstStride;
1204  }
1205  __asm__(EMMS" \n\t"
1206  SFENCE" \n\t"
1207  :::"memory");
1208 }
1209 
1210 /**
1211  * Height should be a multiple of 2 and width should be a multiple of 16.
1212  * (If this is a problem for anyone then tell me, and I will fix it.)
1213  */
1214 static inline void RENAME(yv12toyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1215  int width, int height,
1216  int lumStride, int chromStride, int dstStride)
1217 {
1218  //FIXME interpolate chroma
1219  RENAME(yuvPlanartoyuy2)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 2);
1220 }
1221 
1222 static inline void RENAME(yuvPlanartouyvy)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1223  int width, int height,
1224  int lumStride, int chromStride, int dstStride, int vertLumPerChroma)
1225 {
1226  int y;
1227  const x86_reg chromWidth= width>>1;
1228  for (y=0; y<height; y++) {
1229  //FIXME handle 2 lines at once (fewer prefetches, reuse some chroma, but very likely memory-limited anyway)
1230  __asm__ volatile(
1231  "xor %%"REG_a", %%"REG_a" \n\t"
1232  ".p2align 4 \n\t"
1233  "1: \n\t"
1234  PREFETCH" 32(%1, %%"REG_a", 2) \n\t"
1235  PREFETCH" 32(%2, %%"REG_a") \n\t"
1236  PREFETCH" 32(%3, %%"REG_a") \n\t"
1237  "movq (%2, %%"REG_a"), %%mm0 \n\t" // U(0)
1238  "movq %%mm0, %%mm2 \n\t" // U(0)
1239  "movq (%3, %%"REG_a"), %%mm1 \n\t" // V(0)
1240  "punpcklbw %%mm1, %%mm0 \n\t" // UVUV UVUV(0)
1241  "punpckhbw %%mm1, %%mm2 \n\t" // UVUV UVUV(8)
1242 
1243  "movq (%1, %%"REG_a",2), %%mm3 \n\t" // Y(0)
1244  "movq 8(%1, %%"REG_a",2), %%mm5 \n\t" // Y(8)
1245  "movq %%mm0, %%mm4 \n\t" // Y(0)
1246  "movq %%mm2, %%mm6 \n\t" // Y(8)
1247  "punpcklbw %%mm3, %%mm0 \n\t" // YUYV YUYV(0)
1248  "punpckhbw %%mm3, %%mm4 \n\t" // YUYV YUYV(4)
1249  "punpcklbw %%mm5, %%mm2 \n\t" // YUYV YUYV(8)
1250  "punpckhbw %%mm5, %%mm6 \n\t" // YUYV YUYV(12)
1251 
1252  MOVNTQ" %%mm0, (%0, %%"REG_a", 4) \n\t"
1253  MOVNTQ" %%mm4, 8(%0, %%"REG_a", 4) \n\t"
1254  MOVNTQ" %%mm2, 16(%0, %%"REG_a", 4) \n\t"
1255  MOVNTQ" %%mm6, 24(%0, %%"REG_a", 4) \n\t"
1256 
1257  "add $8, %%"REG_a" \n\t"
1258  "cmp %4, %%"REG_a" \n\t"
1259  " jb 1b \n\t"
1260  ::"r"(dst), "r"(ysrc), "r"(usrc), "r"(vsrc), "g" (chromWidth)
1261  : "%"REG_a
1262  );
1263  if ((y&(vertLumPerChroma-1)) == vertLumPerChroma-1) {
1264  usrc += chromStride;
1265  vsrc += chromStride;
1266  }
1267  ysrc += lumStride;
1268  dst += dstStride;
1269  }
1270  __asm__(EMMS" \n\t"
1271  SFENCE" \n\t"
1272  :::"memory");
1273 }
1274 
1275 /**
1276  * Height should be a multiple of 2 and width should be a multiple of 16
1277  * (If this is a problem for anyone then tell me, and I will fix it.)
1278  */
1279 static inline void RENAME(yv12touyvy)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1280  int width, int height,
1281  int lumStride, int chromStride, int dstStride)
1282 {
1283  //FIXME interpolate chroma
1284  RENAME(yuvPlanartouyvy)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 2);
1285 }
1286 
1287 /**
1288  * Width should be a multiple of 16.
1289  */
1290 static inline void RENAME(yuv422ptouyvy)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1291  int width, int height,
1292  int lumStride, int chromStride, int dstStride)
1293 {
1294  RENAME(yuvPlanartouyvy)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 1);
1295 }
1296 
1297 /**
1298  * Width should be a multiple of 16.
1299  */
1300 static inline void RENAME(yuv422ptoyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1301  int width, int height,
1302  int lumStride, int chromStride, int dstStride)
1303 {
1304  RENAME(yuvPlanartoyuy2)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 1);
1305 }
1306 
1307 /**
1308  * Height should be a multiple of 2 and width should be a multiple of 16.
1309  * (If this is a problem for anyone then tell me, and I will fix it.)
1310  */
1311 static inline void RENAME(yuy2toyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
1312  int width, int height,
1313  int lumStride, int chromStride, int srcStride)
1314 {
1315  int y;
1316  const x86_reg chromWidth= width>>1;
1317  for (y=0; y<height; y+=2) {
1318  __asm__ volatile(
1319  "xor %%"REG_a", %%"REG_a" \n\t"
1320  "pcmpeqw %%mm7, %%mm7 \n\t"
1321  "psrlw $8, %%mm7 \n\t" // FF,00,FF,00...
1322  ".p2align 4 \n\t"
1323  "1: \n\t"
1324  PREFETCH" 64(%0, %%"REG_a", 4) \n\t"
1325  "movq (%0, %%"REG_a", 4), %%mm0 \n\t" // YUYV YUYV(0)
1326  "movq 8(%0, %%"REG_a", 4), %%mm1 \n\t" // YUYV YUYV(4)
1327  "movq %%mm0, %%mm2 \n\t" // YUYV YUYV(0)
1328  "movq %%mm1, %%mm3 \n\t" // YUYV YUYV(4)
1329  "psrlw $8, %%mm0 \n\t" // U0V0 U0V0(0)
1330  "psrlw $8, %%mm1 \n\t" // U0V0 U0V0(4)
1331  "pand %%mm7, %%mm2 \n\t" // Y0Y0 Y0Y0(0)
1332  "pand %%mm7, %%mm3 \n\t" // Y0Y0 Y0Y0(4)
1333  "packuswb %%mm1, %%mm0 \n\t" // UVUV UVUV(0)
1334  "packuswb %%mm3, %%mm2 \n\t" // YYYY YYYY(0)
1335 
1336  MOVNTQ" %%mm2, (%1, %%"REG_a", 2) \n\t"
1337 
1338  "movq 16(%0, %%"REG_a", 4), %%mm1 \n\t" // YUYV YUYV(8)
1339  "movq 24(%0, %%"REG_a", 4), %%mm2 \n\t" // YUYV YUYV(12)
1340  "movq %%mm1, %%mm3 \n\t" // YUYV YUYV(8)
1341  "movq %%mm2, %%mm4 \n\t" // YUYV YUYV(12)
1342  "psrlw $8, %%mm1 \n\t" // U0V0 U0V0(8)
1343  "psrlw $8, %%mm2 \n\t" // U0V0 U0V0(12)
1344  "pand %%mm7, %%mm3 \n\t" // Y0Y0 Y0Y0(8)
1345  "pand %%mm7, %%mm4 \n\t" // Y0Y0 Y0Y0(12)
1346  "packuswb %%mm2, %%mm1 \n\t" // UVUV UVUV(8)
1347  "packuswb %%mm4, %%mm3 \n\t" // YYYY YYYY(8)
1348 
1349  MOVNTQ" %%mm3, 8(%1, %%"REG_a", 2) \n\t"
1350 
1351  "movq %%mm0, %%mm2 \n\t" // UVUV UVUV(0)
1352  "movq %%mm1, %%mm3 \n\t" // UVUV UVUV(8)
1353  "psrlw $8, %%mm0 \n\t" // V0V0 V0V0(0)
1354  "psrlw $8, %%mm1 \n\t" // V0V0 V0V0(8)
1355  "pand %%mm7, %%mm2 \n\t" // U0U0 U0U0(0)
1356  "pand %%mm7, %%mm3 \n\t" // U0U0 U0U0(8)
1357  "packuswb %%mm1, %%mm0 \n\t" // VVVV VVVV(0)
1358  "packuswb %%mm3, %%mm2 \n\t" // UUUU UUUU(0)
1359 
1360  MOVNTQ" %%mm0, (%3, %%"REG_a") \n\t"
1361  MOVNTQ" %%mm2, (%2, %%"REG_a") \n\t"
1362 
1363  "add $8, %%"REG_a" \n\t"
1364  "cmp %4, %%"REG_a" \n\t"
1365  " jb 1b \n\t"
1366  ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
1367  : "memory", "%"REG_a
1368  );
1369 
1370  ydst += lumStride;
1371  src += srcStride;
1372 
1373  __asm__ volatile(
1374  "xor %%"REG_a", %%"REG_a" \n\t"
1375  ".p2align 4 \n\t"
1376  "1: \n\t"
1377  PREFETCH" 64(%0, %%"REG_a", 4) \n\t"
1378  "movq (%0, %%"REG_a", 4), %%mm0 \n\t" // YUYV YUYV(0)
1379  "movq 8(%0, %%"REG_a", 4), %%mm1 \n\t" // YUYV YUYV(4)
1380  "movq 16(%0, %%"REG_a", 4), %%mm2 \n\t" // YUYV YUYV(8)
1381  "movq 24(%0, %%"REG_a", 4), %%mm3 \n\t" // YUYV YUYV(12)
1382  "pand %%mm7, %%mm0 \n\t" // Y0Y0 Y0Y0(0)
1383  "pand %%mm7, %%mm1 \n\t" // Y0Y0 Y0Y0(4)
1384  "pand %%mm7, %%mm2 \n\t" // Y0Y0 Y0Y0(8)
1385  "pand %%mm7, %%mm3 \n\t" // Y0Y0 Y0Y0(12)
1386  "packuswb %%mm1, %%mm0 \n\t" // YYYY YYYY(0)
1387  "packuswb %%mm3, %%mm2 \n\t" // YYYY YYYY(8)
1388 
1389  MOVNTQ" %%mm0, (%1, %%"REG_a", 2) \n\t"
1390  MOVNTQ" %%mm2, 8(%1, %%"REG_a", 2) \n\t"
1391 
1392  "add $8, %%"REG_a" \n\t"
1393  "cmp %4, %%"REG_a" \n\t"
1394  " jb 1b \n\t"
1395 
1396  ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
1397  : "memory", "%"REG_a
1398  );
1399  udst += chromStride;
1400  vdst += chromStride;
1401  ydst += lumStride;
1402  src += srcStride;
1403  }
1404  __asm__ volatile(EMMS" \n\t"
1405  SFENCE" \n\t"
1406  :::"memory");
1407 }
1408 #endif /* !COMPILE_TEMPLATE_AMD3DNOW */
1409 
1410 #if COMPILE_TEMPLATE_MMXEXT || COMPILE_TEMPLATE_AMD3DNOW
1411 static inline void RENAME(planar2x)(const uint8_t *src, uint8_t *dst, int srcWidth, int srcHeight, int srcStride, int dstStride)
1412 {
1413  int x,y;
1414 
1415  dst[0]= src[0];
1416 
1417  // first line
1418  for (x=0; x<srcWidth-1; x++) {
1419  dst[2*x+1]= (3*src[x] + src[x+1])>>2;
1420  dst[2*x+2]= ( src[x] + 3*src[x+1])>>2;
1421  }
1422  dst[2*srcWidth-1]= src[srcWidth-1];
1423 
1424  dst+= dstStride;
1425 
1426  for (y=1; y<srcHeight; y++) {
1427  const x86_reg mmxSize= srcWidth&~15;
1428  __asm__ volatile(
1429  "mov %4, %%"REG_a" \n\t"
1430  "movq "MANGLE(mmx_ff)", %%mm0 \n\t"
1431  "movq (%0, %%"REG_a"), %%mm4 \n\t"
1432  "movq %%mm4, %%mm2 \n\t"
1433  "psllq $8, %%mm4 \n\t"
1434  "pand %%mm0, %%mm2 \n\t"
1435  "por %%mm2, %%mm4 \n\t"
1436  "movq (%1, %%"REG_a"), %%mm5 \n\t"
1437  "movq %%mm5, %%mm3 \n\t"
1438  "psllq $8, %%mm5 \n\t"
1439  "pand %%mm0, %%mm3 \n\t"
1440  "por %%mm3, %%mm5 \n\t"
1441  "1: \n\t"
1442  "movq (%0, %%"REG_a"), %%mm0 \n\t"
1443  "movq (%1, %%"REG_a"), %%mm1 \n\t"
1444  "movq 1(%0, %%"REG_a"), %%mm2 \n\t"
1445  "movq 1(%1, %%"REG_a"), %%mm3 \n\t"
1446  PAVGB" %%mm0, %%mm5 \n\t"
1447  PAVGB" %%mm0, %%mm3 \n\t"
1448  PAVGB" %%mm0, %%mm5 \n\t"
1449  PAVGB" %%mm0, %%mm3 \n\t"
1450  PAVGB" %%mm1, %%mm4 \n\t"
1451  PAVGB" %%mm1, %%mm2 \n\t"
1452  PAVGB" %%mm1, %%mm4 \n\t"
1453  PAVGB" %%mm1, %%mm2 \n\t"
1454  "movq %%mm5, %%mm7 \n\t"
1455  "movq %%mm4, %%mm6 \n\t"
1456  "punpcklbw %%mm3, %%mm5 \n\t"
1457  "punpckhbw %%mm3, %%mm7 \n\t"
1458  "punpcklbw %%mm2, %%mm4 \n\t"
1459  "punpckhbw %%mm2, %%mm6 \n\t"
1460  MOVNTQ" %%mm5, (%2, %%"REG_a", 2) \n\t"
1461  MOVNTQ" %%mm7, 8(%2, %%"REG_a", 2) \n\t"
1462  MOVNTQ" %%mm4, (%3, %%"REG_a", 2) \n\t"
1463  MOVNTQ" %%mm6, 8(%3, %%"REG_a", 2) \n\t"
1464  "add $8, %%"REG_a" \n\t"
1465  "movq -1(%0, %%"REG_a"), %%mm4 \n\t"
1466  "movq -1(%1, %%"REG_a"), %%mm5 \n\t"
1467  " js 1b \n\t"
1468  :: "r" (src + mmxSize ), "r" (src + srcStride + mmxSize ),
1469  "r" (dst + mmxSize*2), "r" (dst + dstStride + mmxSize*2),
1470  "g" (-mmxSize)
1471  : "%"REG_a
1472  );
1473 
1474  for (x=mmxSize-1; x<srcWidth-1; x++) {
1475  dst[2*x +1]= (3*src[x+0] + src[x+srcStride+1])>>2;
1476  dst[2*x+dstStride+2]= ( src[x+0] + 3*src[x+srcStride+1])>>2;
1477  dst[2*x+dstStride+1]= ( src[x+1] + 3*src[x+srcStride ])>>2;
1478  dst[2*x +2]= (3*src[x+1] + src[x+srcStride ])>>2;
1479  }
1480  dst[srcWidth*2 -1 ]= (3*src[srcWidth-1] + src[srcWidth-1 + srcStride])>>2;
1481  dst[srcWidth*2 -1 + dstStride]= ( src[srcWidth-1] + 3*src[srcWidth-1 + srcStride])>>2;
1482 
1483  dst+=dstStride*2;
1484  src+=srcStride;
1485  }
1486 
1487  // last line
1488  dst[0]= src[0];
1489 
1490  for (x=0; x<srcWidth-1; x++) {
1491  dst[2*x+1]= (3*src[x] + src[x+1])>>2;
1492  dst[2*x+2]= ( src[x] + 3*src[x+1])>>2;
1493  }
1494  dst[2*srcWidth-1]= src[srcWidth-1];
1495 
1496  __asm__ volatile(EMMS" \n\t"
1497  SFENCE" \n\t"
1498  :::"memory");
1499 }
1500 #endif /* COMPILE_TEMPLATE_MMXEXT || COMPILE_TEMPLATE_AMD3DNOW */
1501 
1502 #if !COMPILE_TEMPLATE_AMD3DNOW
1503 /**
1504  * Height should be a multiple of 2 and width should be a multiple of 16.
1505  * (If this is a problem for anyone then tell me, and I will fix it.)
1506  * Chrominance data is only taken from every second line, others are ignored.
1507  * FIXME: Write HQ version.
1508  */
1509 static inline void RENAME(uyvytoyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
1510  int width, int height,
1511  int lumStride, int chromStride, int srcStride)
1512 {
1513  int y;
1514  const x86_reg chromWidth= width>>1;
1515  for (y=0; y<height; y+=2) {
1516  __asm__ volatile(
1517  "xor %%"REG_a", %%"REG_a" \n\t"
1518  "pcmpeqw %%mm7, %%mm7 \n\t"
1519  "psrlw $8, %%mm7 \n\t" // FF,00,FF,00...
1520  ".p2align 4 \n\t"
1521  "1: \n\t"
1522  PREFETCH" 64(%0, %%"REG_a", 4) \n\t"
1523  "movq (%0, %%"REG_a", 4), %%mm0 \n\t" // UYVY UYVY(0)
1524  "movq 8(%0, %%"REG_a", 4), %%mm1 \n\t" // UYVY UYVY(4)
1525  "movq %%mm0, %%mm2 \n\t" // UYVY UYVY(0)
1526  "movq %%mm1, %%mm3 \n\t" // UYVY UYVY(4)
1527  "pand %%mm7, %%mm0 \n\t" // U0V0 U0V0(0)
1528  "pand %%mm7, %%mm1 \n\t" // U0V0 U0V0(4)
1529  "psrlw $8, %%mm2 \n\t" // Y0Y0 Y0Y0(0)
1530  "psrlw $8, %%mm3 \n\t" // Y0Y0 Y0Y0(4)
1531  "packuswb %%mm1, %%mm0 \n\t" // UVUV UVUV(0)
1532  "packuswb %%mm3, %%mm2 \n\t" // YYYY YYYY(0)
1533 
1534  MOVNTQ" %%mm2, (%1, %%"REG_a", 2) \n\t"
1535 
1536  "movq 16(%0, %%"REG_a", 4), %%mm1 \n\t" // UYVY UYVY(8)
1537  "movq 24(%0, %%"REG_a", 4), %%mm2 \n\t" // UYVY UYVY(12)
1538  "movq %%mm1, %%mm3 \n\t" // UYVY UYVY(8)
1539  "movq %%mm2, %%mm4 \n\t" // UYVY UYVY(12)
1540  "pand %%mm7, %%mm1 \n\t" // U0V0 U0V0(8)
1541  "pand %%mm7, %%mm2 \n\t" // U0V0 U0V0(12)
1542  "psrlw $8, %%mm3 \n\t" // Y0Y0 Y0Y0(8)
1543  "psrlw $8, %%mm4 \n\t" // Y0Y0 Y0Y0(12)
1544  "packuswb %%mm2, %%mm1 \n\t" // UVUV UVUV(8)
1545  "packuswb %%mm4, %%mm3 \n\t" // YYYY YYYY(8)
1546 
1547  MOVNTQ" %%mm3, 8(%1, %%"REG_a", 2) \n\t"
1548 
1549  "movq %%mm0, %%mm2 \n\t" // UVUV UVUV(0)
1550  "movq %%mm1, %%mm3 \n\t" // UVUV UVUV(8)
1551  "psrlw $8, %%mm0 \n\t" // V0V0 V0V0(0)
1552  "psrlw $8, %%mm1 \n\t" // V0V0 V0V0(8)
1553  "pand %%mm7, %%mm2 \n\t" // U0U0 U0U0(0)
1554  "pand %%mm7, %%mm3 \n\t" // U0U0 U0U0(8)
1555  "packuswb %%mm1, %%mm0 \n\t" // VVVV VVVV(0)
1556  "packuswb %%mm3, %%mm2 \n\t" // UUUU UUUU(0)
1557 
1558  MOVNTQ" %%mm0, (%3, %%"REG_a") \n\t"
1559  MOVNTQ" %%mm2, (%2, %%"REG_a") \n\t"
1560 
1561  "add $8, %%"REG_a" \n\t"
1562  "cmp %4, %%"REG_a" \n\t"
1563  " jb 1b \n\t"
1564  ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
1565  : "memory", "%"REG_a
1566  );
1567 
1568  ydst += lumStride;
1569  src += srcStride;
1570 
1571  __asm__ volatile(
1572  "xor %%"REG_a", %%"REG_a" \n\t"
1573  ".p2align 4 \n\t"
1574  "1: \n\t"
1575  PREFETCH" 64(%0, %%"REG_a", 4) \n\t"
1576  "movq (%0, %%"REG_a", 4), %%mm0 \n\t" // YUYV YUYV(0)
1577  "movq 8(%0, %%"REG_a", 4), %%mm1 \n\t" // YUYV YUYV(4)
1578  "movq 16(%0, %%"REG_a", 4), %%mm2 \n\t" // YUYV YUYV(8)
1579  "movq 24(%0, %%"REG_a", 4), %%mm3 \n\t" // YUYV YUYV(12)
1580  "psrlw $8, %%mm0 \n\t" // Y0Y0 Y0Y0(0)
1581  "psrlw $8, %%mm1 \n\t" // Y0Y0 Y0Y0(4)
1582  "psrlw $8, %%mm2 \n\t" // Y0Y0 Y0Y0(8)
1583  "psrlw $8, %%mm3 \n\t" // Y0Y0 Y0Y0(12)
1584  "packuswb %%mm1, %%mm0 \n\t" // YYYY YYYY(0)
1585  "packuswb %%mm3, %%mm2 \n\t" // YYYY YYYY(8)
1586 
1587  MOVNTQ" %%mm0, (%1, %%"REG_a", 2) \n\t"
1588  MOVNTQ" %%mm2, 8(%1, %%"REG_a", 2) \n\t"
1589 
1590  "add $8, %%"REG_a" \n\t"
1591  "cmp %4, %%"REG_a" \n\t"
1592  " jb 1b \n\t"
1593 
1594  ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
1595  : "memory", "%"REG_a
1596  );
1597  udst += chromStride;
1598  vdst += chromStride;
1599  ydst += lumStride;
1600  src += srcStride;
1601  }
1602  __asm__ volatile(EMMS" \n\t"
1603  SFENCE" \n\t"
1604  :::"memory");
1605 }
1606 #endif /* !COMPILE_TEMPLATE_AMD3DNOW */
1607 
1608 /**
1609  * Height should be a multiple of 2 and width should be a multiple of 2.
1610  * (If this is a problem for anyone then tell me, and I will fix it.)
1611  * Chrominance data is only taken from every second line,
1612  * others are ignored in the C version.
1613  * FIXME: Write HQ version.
1614  */
1615 #if HAVE_7REGS
1616 static inline void RENAME(rgb24toyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
1617  int width, int height,
1618  int lumStride, int chromStride, int srcStride,
1619  int32_t *rgb2yuv)
1620 {
1621 #define BGR2Y_IDX "16*4+16*32"
1622 #define BGR2U_IDX "16*4+16*33"
1623 #define BGR2V_IDX "16*4+16*34"
1624  int y;
1625  const x86_reg chromWidth= width>>1;
1626  for (y=0; y<height-2; y+=2) {
1627  int i;
1628  for (i=0; i<2; i++) {
1629  __asm__ volatile(
1630  "mov %2, %%"REG_a" \n\t"
1631  "movq "BGR2Y_IDX"(%3), %%mm6 \n\t"
1632  "movq "MANGLE(ff_w1111)", %%mm5 \n\t"
1633  "pxor %%mm7, %%mm7 \n\t"
1634  "lea (%%"REG_a", %%"REG_a", 2), %%"REG_d" \n\t"
1635  ".p2align 4 \n\t"
1636  "1: \n\t"
1637  PREFETCH" 64(%0, %%"REG_d") \n\t"
1638  "movd (%0, %%"REG_d"), %%mm0 \n\t"
1639  "movd 3(%0, %%"REG_d"), %%mm1 \n\t"
1640  "punpcklbw %%mm7, %%mm0 \n\t"
1641  "punpcklbw %%mm7, %%mm1 \n\t"
1642  "movd 6(%0, %%"REG_d"), %%mm2 \n\t"
1643  "movd 9(%0, %%"REG_d"), %%mm3 \n\t"
1644  "punpcklbw %%mm7, %%mm2 \n\t"
1645  "punpcklbw %%mm7, %%mm3 \n\t"
1646  "pmaddwd %%mm6, %%mm0 \n\t"
1647  "pmaddwd %%mm6, %%mm1 \n\t"
1648  "pmaddwd %%mm6, %%mm2 \n\t"
1649  "pmaddwd %%mm6, %%mm3 \n\t"
1650  "psrad $8, %%mm0 \n\t"
1651  "psrad $8, %%mm1 \n\t"
1652  "psrad $8, %%mm2 \n\t"
1653  "psrad $8, %%mm3 \n\t"
1654  "packssdw %%mm1, %%mm0 \n\t"
1655  "packssdw %%mm3, %%mm2 \n\t"
1656  "pmaddwd %%mm5, %%mm0 \n\t"
1657  "pmaddwd %%mm5, %%mm2 \n\t"
1658  "packssdw %%mm2, %%mm0 \n\t"
1659  "psraw $7, %%mm0 \n\t"
1660 
1661  "movd 12(%0, %%"REG_d"), %%mm4 \n\t"
1662  "movd 15(%0, %%"REG_d"), %%mm1 \n\t"
1663  "punpcklbw %%mm7, %%mm4 \n\t"
1664  "punpcklbw %%mm7, %%mm1 \n\t"
1665  "movd 18(%0, %%"REG_d"), %%mm2 \n\t"
1666  "movd 21(%0, %%"REG_d"), %%mm3 \n\t"
1667  "punpcklbw %%mm7, %%mm2 \n\t"
1668  "punpcklbw %%mm7, %%mm3 \n\t"
1669  "pmaddwd %%mm6, %%mm4 \n\t"
1670  "pmaddwd %%mm6, %%mm1 \n\t"
1671  "pmaddwd %%mm6, %%mm2 \n\t"
1672  "pmaddwd %%mm6, %%mm3 \n\t"
1673  "psrad $8, %%mm4 \n\t"
1674  "psrad $8, %%mm1 \n\t"
1675  "psrad $8, %%mm2 \n\t"
1676  "psrad $8, %%mm3 \n\t"
1677  "packssdw %%mm1, %%mm4 \n\t"
1678  "packssdw %%mm3, %%mm2 \n\t"
1679  "pmaddwd %%mm5, %%mm4 \n\t"
1680  "pmaddwd %%mm5, %%mm2 \n\t"
1681  "add $24, %%"REG_d" \n\t"
1682  "packssdw %%mm2, %%mm4 \n\t"
1683  "psraw $7, %%mm4 \n\t"
1684 
1685  "packuswb %%mm4, %%mm0 \n\t"
1686  "paddusb "MANGLE(ff_bgr2YOffset)", %%mm0 \n\t"
1687 
1688  MOVNTQ" %%mm0, (%1, %%"REG_a") \n\t"
1689  "add $8, %%"REG_a" \n\t"
1690  " js 1b \n\t"
1691  : : "r" (src+width*3), "r" (ydst+width), "g" ((x86_reg)-width), "r"(rgb2yuv)
1692  : "%"REG_a, "%"REG_d
1693  );
1694  ydst += lumStride;
1695  src += srcStride;
1696  }
1697  src -= srcStride*2;
1698  __asm__ volatile(
1699  "mov %4, %%"REG_a" \n\t"
1700  "movq "MANGLE(ff_w1111)", %%mm5 \n\t"
1701  "movq "BGR2U_IDX"(%5), %%mm6 \n\t"
1702  "pxor %%mm7, %%mm7 \n\t"
1703  "lea (%%"REG_a", %%"REG_a", 2), %%"REG_d" \n\t"
1704  "add %%"REG_d", %%"REG_d" \n\t"
1705  ".p2align 4 \n\t"
1706  "1: \n\t"
1707  PREFETCH" 64(%0, %%"REG_d") \n\t"
1708  PREFETCH" 64(%1, %%"REG_d") \n\t"
1709 #if COMPILE_TEMPLATE_MMXEXT || COMPILE_TEMPLATE_AMD3DNOW
1710  "movq (%0, %%"REG_d"), %%mm0 \n\t"
1711  "movq (%1, %%"REG_d"), %%mm1 \n\t"
1712  "movq 6(%0, %%"REG_d"), %%mm2 \n\t"
1713  "movq 6(%1, %%"REG_d"), %%mm3 \n\t"
1714  PAVGB" %%mm1, %%mm0 \n\t"
1715  PAVGB" %%mm3, %%mm2 \n\t"
1716  "movq %%mm0, %%mm1 \n\t"
1717  "movq %%mm2, %%mm3 \n\t"
1718  "psrlq $24, %%mm0 \n\t"
1719  "psrlq $24, %%mm2 \n\t"
1720  PAVGB" %%mm1, %%mm0 \n\t"
1721  PAVGB" %%mm3, %%mm2 \n\t"
1722  "punpcklbw %%mm7, %%mm0 \n\t"
1723  "punpcklbw %%mm7, %%mm2 \n\t"
1724 #else
1725  "movd (%0, %%"REG_d"), %%mm0 \n\t"
1726  "movd (%1, %%"REG_d"), %%mm1 \n\t"
1727  "movd 3(%0, %%"REG_d"), %%mm2 \n\t"
1728  "movd 3(%1, %%"REG_d"), %%mm3 \n\t"
1729  "punpcklbw %%mm7, %%mm0 \n\t"
1730  "punpcklbw %%mm7, %%mm1 \n\t"
1731  "punpcklbw %%mm7, %%mm2 \n\t"
1732  "punpcklbw %%mm7, %%mm3 \n\t"
1733  "paddw %%mm1, %%mm0 \n\t"
1734  "paddw %%mm3, %%mm2 \n\t"
1735  "paddw %%mm2, %%mm0 \n\t"
1736  "movd 6(%0, %%"REG_d"), %%mm4 \n\t"
1737  "movd 6(%1, %%"REG_d"), %%mm1 \n\t"
1738  "movd 9(%0, %%"REG_d"), %%mm2 \n\t"
1739  "movd 9(%1, %%"REG_d"), %%mm3 \n\t"
1740  "punpcklbw %%mm7, %%mm4 \n\t"
1741  "punpcklbw %%mm7, %%mm1 \n\t"
1742  "punpcklbw %%mm7, %%mm2 \n\t"
1743  "punpcklbw %%mm7, %%mm3 \n\t"
1744  "paddw %%mm1, %%mm4 \n\t"
1745  "paddw %%mm3, %%mm2 \n\t"
1746  "paddw %%mm4, %%mm2 \n\t"
1747  "psrlw $2, %%mm0 \n\t"
1748  "psrlw $2, %%mm2 \n\t"
1749 #endif
1750  "movq "BGR2V_IDX"(%5), %%mm1 \n\t"
1751  "movq "BGR2V_IDX"(%5), %%mm3 \n\t"
1752 
1753  "pmaddwd %%mm0, %%mm1 \n\t"
1754  "pmaddwd %%mm2, %%mm3 \n\t"
1755  "pmaddwd %%mm6, %%mm0 \n\t"
1756  "pmaddwd %%mm6, %%mm2 \n\t"
1757  "psrad $8, %%mm0 \n\t"
1758  "psrad $8, %%mm1 \n\t"
1759  "psrad $8, %%mm2 \n\t"
1760  "psrad $8, %%mm3 \n\t"
1761  "packssdw %%mm2, %%mm0 \n\t"
1762  "packssdw %%mm3, %%mm1 \n\t"
1763  "pmaddwd %%mm5, %%mm0 \n\t"
1764  "pmaddwd %%mm5, %%mm1 \n\t"
1765  "packssdw %%mm1, %%mm0 \n\t" // V1 V0 U1 U0
1766  "psraw $7, %%mm0 \n\t"
1767 
1768 #if COMPILE_TEMPLATE_MMXEXT || COMPILE_TEMPLATE_AMD3DNOW
1769  "movq 12(%0, %%"REG_d"), %%mm4 \n\t"
1770  "movq 12(%1, %%"REG_d"), %%mm1 \n\t"
1771  "movq 18(%0, %%"REG_d"), %%mm2 \n\t"
1772  "movq 18(%1, %%"REG_d"), %%mm3 \n\t"
1773  PAVGB" %%mm1, %%mm4 \n\t"
1774  PAVGB" %%mm3, %%mm2 \n\t"
1775  "movq %%mm4, %%mm1 \n\t"
1776  "movq %%mm2, %%mm3 \n\t"
1777  "psrlq $24, %%mm4 \n\t"
1778  "psrlq $24, %%mm2 \n\t"
1779  PAVGB" %%mm1, %%mm4 \n\t"
1780  PAVGB" %%mm3, %%mm2 \n\t"
1781  "punpcklbw %%mm7, %%mm4 \n\t"
1782  "punpcklbw %%mm7, %%mm2 \n\t"
1783 #else
1784  "movd 12(%0, %%"REG_d"), %%mm4 \n\t"
1785  "movd 12(%1, %%"REG_d"), %%mm1 \n\t"
1786  "movd 15(%0, %%"REG_d"), %%mm2 \n\t"
1787  "movd 15(%1, %%"REG_d"), %%mm3 \n\t"
1788  "punpcklbw %%mm7, %%mm4 \n\t"
1789  "punpcklbw %%mm7, %%mm1 \n\t"
1790  "punpcklbw %%mm7, %%mm2 \n\t"
1791  "punpcklbw %%mm7, %%mm3 \n\t"
1792  "paddw %%mm1, %%mm4 \n\t"
1793  "paddw %%mm3, %%mm2 \n\t"
1794  "paddw %%mm2, %%mm4 \n\t"
1795  "movd 18(%0, %%"REG_d"), %%mm5 \n\t"
1796  "movd 18(%1, %%"REG_d"), %%mm1 \n\t"
1797  "movd 21(%0, %%"REG_d"), %%mm2 \n\t"
1798  "movd 21(%1, %%"REG_d"), %%mm3 \n\t"
1799  "punpcklbw %%mm7, %%mm5 \n\t"
1800  "punpcklbw %%mm7, %%mm1 \n\t"
1801  "punpcklbw %%mm7, %%mm2 \n\t"
1802  "punpcklbw %%mm7, %%mm3 \n\t"
1803  "paddw %%mm1, %%mm5 \n\t"
1804  "paddw %%mm3, %%mm2 \n\t"
1805  "paddw %%mm5, %%mm2 \n\t"
1806  "movq "MANGLE(ff_w1111)", %%mm5 \n\t"
1807  "psrlw $2, %%mm4 \n\t"
1808  "psrlw $2, %%mm2 \n\t"
1809 #endif
1810  "movq "BGR2V_IDX"(%5), %%mm1 \n\t"
1811  "movq "BGR2V_IDX"(%5), %%mm3 \n\t"
1812 
1813  "pmaddwd %%mm4, %%mm1 \n\t"
1814  "pmaddwd %%mm2, %%mm3 \n\t"
1815  "pmaddwd %%mm6, %%mm4 \n\t"
1816  "pmaddwd %%mm6, %%mm2 \n\t"
1817  "psrad $8, %%mm4 \n\t"
1818  "psrad $8, %%mm1 \n\t"
1819  "psrad $8, %%mm2 \n\t"
1820  "psrad $8, %%mm3 \n\t"
1821  "packssdw %%mm2, %%mm4 \n\t"
1822  "packssdw %%mm3, %%mm1 \n\t"
1823  "pmaddwd %%mm5, %%mm4 \n\t"
1824  "pmaddwd %%mm5, %%mm1 \n\t"
1825  "add $24, %%"REG_d" \n\t"
1826  "packssdw %%mm1, %%mm4 \n\t" // V3 V2 U3 U2
1827  "psraw $7, %%mm4 \n\t"
1828 
1829  "movq %%mm0, %%mm1 \n\t"
1830  "punpckldq %%mm4, %%mm0 \n\t"
1831  "punpckhdq %%mm4, %%mm1 \n\t"
1832  "packsswb %%mm1, %%mm0 \n\t"
1833  "paddb "MANGLE(ff_bgr2UVOffset)", %%mm0 \n\t"
1834  "movd %%mm0, (%2, %%"REG_a") \n\t"
1835  "punpckhdq %%mm0, %%mm0 \n\t"
1836  "movd %%mm0, (%3, %%"REG_a") \n\t"
1837  "add $4, %%"REG_a" \n\t"
1838  " js 1b \n\t"
1839  : : "r" (src+chromWidth*6), "r" (src+srcStride+chromWidth*6), "r" (udst+chromWidth), "r" (vdst+chromWidth), "g" (-chromWidth), "r"(rgb2yuv)
1840  : "%"REG_a, "%"REG_d
1841  );
1842 
1843  udst += chromStride;
1844  vdst += chromStride;
1845  src += srcStride*2;
1846  }
1847 
1848  __asm__ volatile(EMMS" \n\t"
1849  SFENCE" \n\t"
1850  :::"memory");
1851 
1852  ff_rgb24toyv12_c(src, ydst, udst, vdst, width, height-y, lumStride, chromStride, srcStride, rgb2yuv);
1853 }
1854 #endif /* HAVE_7REGS */
1855 #endif /* !COMPILE_TEMPLATE_SSE2 */
1856 
1857 #if !COMPILE_TEMPLATE_AMD3DNOW && !COMPILE_TEMPLATE_AVX
1858 static void RENAME(interleaveBytes)(const uint8_t *src1, const uint8_t *src2, uint8_t *dest,
1859  int width, int height, int src1Stride,
1860  int src2Stride, int dstStride)
1861 {
1862  int h;
1863 
1864  for (h=0; h < height; h++) {
1865  int w;
1866 
1867 #if COMPILE_TEMPLATE_SSE2
1868  __asm__(
1869  "xor %%"REG_a", %%"REG_a" \n\t"
1870  "1: \n\t"
1871  PREFETCH" 64(%1, %%"REG_a") \n\t"
1872  PREFETCH" 64(%2, %%"REG_a") \n\t"
1873  "movdqa (%1, %%"REG_a"), %%xmm0 \n\t"
1874  "movdqa (%1, %%"REG_a"), %%xmm1 \n\t"
1875  "movdqa (%2, %%"REG_a"), %%xmm2 \n\t"
1876  "punpcklbw %%xmm2, %%xmm0 \n\t"
1877  "punpckhbw %%xmm2, %%xmm1 \n\t"
1878  "movntdq %%xmm0, (%0, %%"REG_a", 2) \n\t"
1879  "movntdq %%xmm1, 16(%0, %%"REG_a", 2) \n\t"
1880  "add $16, %%"REG_a" \n\t"
1881  "cmp %3, %%"REG_a" \n\t"
1882  " jb 1b \n\t"
1883  ::"r"(dest), "r"(src1), "r"(src2), "r" ((x86_reg)width-15)
1884  : "memory", "%"REG_a""
1885  );
1886 #else
1887  __asm__(
1888  "xor %%"REG_a", %%"REG_a" \n\t"
1889  "1: \n\t"
1890  PREFETCH" 64(%1, %%"REG_a") \n\t"
1891  PREFETCH" 64(%2, %%"REG_a") \n\t"
1892  "movq (%1, %%"REG_a"), %%mm0 \n\t"
1893  "movq 8(%1, %%"REG_a"), %%mm2 \n\t"
1894  "movq %%mm0, %%mm1 \n\t"
1895  "movq %%mm2, %%mm3 \n\t"
1896  "movq (%2, %%"REG_a"), %%mm4 \n\t"
1897  "movq 8(%2, %%"REG_a"), %%mm5 \n\t"
1898  "punpcklbw %%mm4, %%mm0 \n\t"
1899  "punpckhbw %%mm4, %%mm1 \n\t"
1900  "punpcklbw %%mm5, %%mm2 \n\t"
1901  "punpckhbw %%mm5, %%mm3 \n\t"
1902  MOVNTQ" %%mm0, (%0, %%"REG_a", 2) \n\t"
1903  MOVNTQ" %%mm1, 8(%0, %%"REG_a", 2) \n\t"
1904  MOVNTQ" %%mm2, 16(%0, %%"REG_a", 2) \n\t"
1905  MOVNTQ" %%mm3, 24(%0, %%"REG_a", 2) \n\t"
1906  "add $16, %%"REG_a" \n\t"
1907  "cmp %3, %%"REG_a" \n\t"
1908  " jb 1b \n\t"
1909  ::"r"(dest), "r"(src1), "r"(src2), "r" ((x86_reg)width-15)
1910  : "memory", "%"REG_a
1911  );
1912 #endif
1913  for (w= (width&(~15)); w < width; w++) {
1914  dest[2*w+0] = src1[w];
1915  dest[2*w+1] = src2[w];
1916  }
1917  dest += dstStride;
1918  src1 += src1Stride;
1919  src2 += src2Stride;
1920  }
1921  __asm__(
1922  EMMS" \n\t"
1923  SFENCE" \n\t"
1924  ::: "memory"
1925  );
1926 }
1927 #endif /* !COMPILE_TEMPLATE_AMD3DNOW && !COMPILE_TEMPLATE_AVX */
1928 
1929 #if !COMPILE_TEMPLATE_AVX || HAVE_AVX_EXTERNAL
1930 #if !COMPILE_TEMPLATE_AMD3DNOW && (ARCH_X86_32 || COMPILE_TEMPLATE_SSE2) && COMPILE_TEMPLATE_MMXEXT == COMPILE_TEMPLATE_SSE2 && HAVE_YASM
1931 void RENAME(ff_nv12ToUV)(uint8_t *dstU, uint8_t *dstV,
1932  const uint8_t *unused,
1933  const uint8_t *src1,
1934  const uint8_t *src2,
1935  int w,
1936  uint32_t *unused2);
1937 static void RENAME(deinterleaveBytes)(const uint8_t *src, uint8_t *dst1, uint8_t *dst2,
1938  int width, int height, int srcStride,
1939  int dst1Stride, int dst2Stride)
1940 {
1941  int h;
1942 
1943  for (h = 0; h < height; h++) {
1944  RENAME(ff_nv12ToUV)(dst1, dst2, NULL, src, NULL, width, NULL);
1945  src += srcStride;
1946  dst1 += dst1Stride;
1947  dst2 += dst2Stride;
1948  }
1949  __asm__(
1950  EMMS" \n\t"
1951  SFENCE" \n\t"
1952  ::: "memory"
1953  );
1954 }
1955 #endif /* !COMPILE_TEMPLATE_AMD3DNOW */
1956 #endif /* !COMPILE_TEMPLATE_AVX || HAVE_AVX_EXTERNAL */
1957 
1958 #if !COMPILE_TEMPLATE_SSE2
1959 #if !COMPILE_TEMPLATE_AMD3DNOW
1960 static inline void RENAME(vu9_to_vu12)(const uint8_t *src1, const uint8_t *src2,
1961  uint8_t *dst1, uint8_t *dst2,
1962  int width, int height,
1963  int srcStride1, int srcStride2,
1964  int dstStride1, int dstStride2)
1965 {
1966  x86_reg x, y;
1967  int w,h;
1968  w=width/2; h=height/2;
1969  __asm__ volatile(
1970  PREFETCH" %0 \n\t"
1971  PREFETCH" %1 \n\t"
1972  ::"m"(*(src1+srcStride1)),"m"(*(src2+srcStride2)):"memory");
1973  for (y=0;y<h;y++) {
1974  const uint8_t* s1=src1+srcStride1*(y>>1);
1975  uint8_t* d=dst1+dstStride1*y;
1976  x=0;
1977  for (;x<w-31;x+=32) {
1978  __asm__ volatile(
1979  PREFETCH" 32(%1,%2) \n\t"
1980  "movq (%1,%2), %%mm0 \n\t"
1981  "movq 8(%1,%2), %%mm2 \n\t"
1982  "movq 16(%1,%2), %%mm4 \n\t"
1983  "movq 24(%1,%2), %%mm6 \n\t"
1984  "movq %%mm0, %%mm1 \n\t"
1985  "movq %%mm2, %%mm3 \n\t"
1986  "movq %%mm4, %%mm5 \n\t"
1987  "movq %%mm6, %%mm7 \n\t"
1988  "punpcklbw %%mm0, %%mm0 \n\t"
1989  "punpckhbw %%mm1, %%mm1 \n\t"
1990  "punpcklbw %%mm2, %%mm2 \n\t"
1991  "punpckhbw %%mm3, %%mm3 \n\t"
1992  "punpcklbw %%mm4, %%mm4 \n\t"
1993  "punpckhbw %%mm5, %%mm5 \n\t"
1994  "punpcklbw %%mm6, %%mm6 \n\t"
1995  "punpckhbw %%mm7, %%mm7 \n\t"
1996  MOVNTQ" %%mm0, (%0,%2,2) \n\t"
1997  MOVNTQ" %%mm1, 8(%0,%2,2) \n\t"
1998  MOVNTQ" %%mm2, 16(%0,%2,2) \n\t"
1999  MOVNTQ" %%mm3, 24(%0,%2,2) \n\t"
2000  MOVNTQ" %%mm4, 32(%0,%2,2) \n\t"
2001  MOVNTQ" %%mm5, 40(%0,%2,2) \n\t"
2002  MOVNTQ" %%mm6, 48(%0,%2,2) \n\t"
2003  MOVNTQ" %%mm7, 56(%0,%2,2)"
2004  :: "r"(d), "r"(s1), "r"(x)
2005  :"memory");
2006  }
2007  for (;x<w;x++) d[2*x]=d[2*x+1]=s1[x];
2008  }
2009  for (y=0;y<h;y++) {
2010  const uint8_t* s2=src2+srcStride2*(y>>1);
2011  uint8_t* d=dst2+dstStride2*y;
2012  x=0;
2013  for (;x<w-31;x+=32) {
2014  __asm__ volatile(
2015  PREFETCH" 32(%1,%2) \n\t"
2016  "movq (%1,%2), %%mm0 \n\t"
2017  "movq 8(%1,%2), %%mm2 \n\t"
2018  "movq 16(%1,%2), %%mm4 \n\t"
2019  "movq 24(%1,%2), %%mm6 \n\t"
2020  "movq %%mm0, %%mm1 \n\t"
2021  "movq %%mm2, %%mm3 \n\t"
2022  "movq %%mm4, %%mm5 \n\t"
2023  "movq %%mm6, %%mm7 \n\t"
2024  "punpcklbw %%mm0, %%mm0 \n\t"
2025  "punpckhbw %%mm1, %%mm1 \n\t"
2026  "punpcklbw %%mm2, %%mm2 \n\t"
2027  "punpckhbw %%mm3, %%mm3 \n\t"
2028  "punpcklbw %%mm4, %%mm4 \n\t"
2029  "punpckhbw %%mm5, %%mm5 \n\t"
2030  "punpcklbw %%mm6, %%mm6 \n\t"
2031  "punpckhbw %%mm7, %%mm7 \n\t"
2032  MOVNTQ" %%mm0, (%0,%2,2) \n\t"
2033  MOVNTQ" %%mm1, 8(%0,%2,2) \n\t"
2034  MOVNTQ" %%mm2, 16(%0,%2,2) \n\t"
2035  MOVNTQ" %%mm3, 24(%0,%2,2) \n\t"
2036  MOVNTQ" %%mm4, 32(%0,%2,2) \n\t"
2037  MOVNTQ" %%mm5, 40(%0,%2,2) \n\t"
2038  MOVNTQ" %%mm6, 48(%0,%2,2) \n\t"
2039  MOVNTQ" %%mm7, 56(%0,%2,2)"
2040  :: "r"(d), "r"(s2), "r"(x)
2041  :"memory");
2042  }
2043  for (;x<w;x++) d[2*x]=d[2*x+1]=s2[x];
2044  }
2045  __asm__(
2046  EMMS" \n\t"
2047  SFENCE" \n\t"
2048  ::: "memory"
2049  );
2050 }
2051 
2052 static inline void RENAME(yvu9_to_yuy2)(const uint8_t *src1, const uint8_t *src2, const uint8_t *src3,
2053  uint8_t *dst,
2054  int width, int height,
2055  int srcStride1, int srcStride2,
2056  int srcStride3, int dstStride)
2057 {
2058  x86_reg x;
2059  int y,w,h;
2060  w=width/2; h=height;
2061  for (y=0;y<h;y++) {
2062  const uint8_t* yp=src1+srcStride1*y;
2063  const uint8_t* up=src2+srcStride2*(y>>2);
2064  const uint8_t* vp=src3+srcStride3*(y>>2);
2065  uint8_t* d=dst+dstStride*y;
2066  x=0;
2067  for (;x<w-7;x+=8) {
2068  __asm__ volatile(
2069  PREFETCH" 32(%1, %0) \n\t"
2070  PREFETCH" 32(%2, %0) \n\t"
2071  PREFETCH" 32(%3, %0) \n\t"
2072  "movq (%1, %0, 4), %%mm0 \n\t" /* Y0Y1Y2Y3Y4Y5Y6Y7 */
2073  "movq (%2, %0), %%mm1 \n\t" /* U0U1U2U3U4U5U6U7 */
2074  "movq (%3, %0), %%mm2 \n\t" /* V0V1V2V3V4V5V6V7 */
2075  "movq %%mm0, %%mm3 \n\t" /* Y0Y1Y2Y3Y4Y5Y6Y7 */
2076  "movq %%mm1, %%mm4 \n\t" /* U0U1U2U3U4U5U6U7 */
2077  "movq %%mm2, %%mm5 \n\t" /* V0V1V2V3V4V5V6V7 */
2078  "punpcklbw %%mm1, %%mm1 \n\t" /* U0U0 U1U1 U2U2 U3U3 */
2079  "punpcklbw %%mm2, %%mm2 \n\t" /* V0V0 V1V1 V2V2 V3V3 */
2080  "punpckhbw %%mm4, %%mm4 \n\t" /* U4U4 U5U5 U6U6 U7U7 */
2081  "punpckhbw %%mm5, %%mm5 \n\t" /* V4V4 V5V5 V6V6 V7V7 */
2082 
2083  "movq %%mm1, %%mm6 \n\t"
2084  "punpcklbw %%mm2, %%mm1 \n\t" /* U0V0 U0V0 U1V1 U1V1*/
2085  "punpcklbw %%mm1, %%mm0 \n\t" /* Y0U0 Y1V0 Y2U0 Y3V0*/
2086  "punpckhbw %%mm1, %%mm3 \n\t" /* Y4U1 Y5V1 Y6U1 Y7V1*/
2087  MOVNTQ" %%mm0, (%4, %0, 8) \n\t"
2088  MOVNTQ" %%mm3, 8(%4, %0, 8) \n\t"
2089 
2090  "punpckhbw %%mm2, %%mm6 \n\t" /* U2V2 U2V2 U3V3 U3V3*/
2091  "movq 8(%1, %0, 4), %%mm0 \n\t"
2092  "movq %%mm0, %%mm3 \n\t"
2093  "punpcklbw %%mm6, %%mm0 \n\t" /* Y U2 Y V2 Y U2 Y V2*/
2094  "punpckhbw %%mm6, %%mm3 \n\t" /* Y U3 Y V3 Y U3 Y V3*/
2095  MOVNTQ" %%mm0, 16(%4, %0, 8) \n\t"
2096  MOVNTQ" %%mm3, 24(%4, %0, 8) \n\t"
2097 
2098  "movq %%mm4, %%mm6 \n\t"
2099  "movq 16(%1, %0, 4), %%mm0 \n\t"
2100  "movq %%mm0, %%mm3 \n\t"
2101  "punpcklbw %%mm5, %%mm4 \n\t"
2102  "punpcklbw %%mm4, %%mm0 \n\t" /* Y U4 Y V4 Y U4 Y V4*/
2103  "punpckhbw %%mm4, %%mm3 \n\t" /* Y U5 Y V5 Y U5 Y V5*/
2104  MOVNTQ" %%mm0, 32(%4, %0, 8) \n\t"
2105  MOVNTQ" %%mm3, 40(%4, %0, 8) \n\t"
2106 
2107  "punpckhbw %%mm5, %%mm6 \n\t"
2108  "movq 24(%1, %0, 4), %%mm0 \n\t"
2109  "movq %%mm0, %%mm3 \n\t"
2110  "punpcklbw %%mm6, %%mm0 \n\t" /* Y U6 Y V6 Y U6 Y V6*/
2111  "punpckhbw %%mm6, %%mm3 \n\t" /* Y U7 Y V7 Y U7 Y V7*/
2112  MOVNTQ" %%mm0, 48(%4, %0, 8) \n\t"
2113  MOVNTQ" %%mm3, 56(%4, %0, 8) \n\t"
2114 
2115  : "+r" (x)
2116  : "r"(yp), "r" (up), "r"(vp), "r"(d)
2117  :"memory");
2118  }
2119  for (; x<w; x++) {
2120  const int x2 = x<<2;
2121  d[8*x+0] = yp[x2];
2122  d[8*x+1] = up[x];
2123  d[8*x+2] = yp[x2+1];
2124  d[8*x+3] = vp[x];
2125  d[8*x+4] = yp[x2+2];
2126  d[8*x+5] = up[x];
2127  d[8*x+6] = yp[x2+3];
2128  d[8*x+7] = vp[x];
2129  }
2130  }
2131  __asm__(
2132  EMMS" \n\t"
2133  SFENCE" \n\t"
2134  ::: "memory"
2135  );
2136 }
2137 #endif /* !COMPILE_TEMPLATE_AMD3DNOW */
2138 
2139 static void RENAME(extract_even)(const uint8_t *src, uint8_t *dst, x86_reg count)
2140 {
2141  dst += count;
2142  src += 2*count;
2143  count= - count;
2144 
2145  if(count <= -16) {
2146  count += 15;
2147  __asm__ volatile(
2148  "pcmpeqw %%mm7, %%mm7 \n\t"
2149  "psrlw $8, %%mm7 \n\t"
2150  "1: \n\t"
2151  "movq -30(%1, %0, 2), %%mm0 \n\t"
2152  "movq -22(%1, %0, 2), %%mm1 \n\t"
2153  "movq -14(%1, %0, 2), %%mm2 \n\t"
2154  "movq -6(%1, %0, 2), %%mm3 \n\t"
2155  "pand %%mm7, %%mm0 \n\t"
2156  "pand %%mm7, %%mm1 \n\t"
2157  "pand %%mm7, %%mm2 \n\t"
2158  "pand %%mm7, %%mm3 \n\t"
2159  "packuswb %%mm1, %%mm0 \n\t"
2160  "packuswb %%mm3, %%mm2 \n\t"
2161  MOVNTQ" %%mm0,-15(%2, %0) \n\t"
2162  MOVNTQ" %%mm2,- 7(%2, %0) \n\t"
2163  "add $16, %0 \n\t"
2164  " js 1b \n\t"
2165  : "+r"(count)
2166  : "r"(src), "r"(dst)
2167  );
2168  count -= 15;
2169  }
2170  while(count<0) {
2171  dst[count]= src[2*count];
2172  count++;
2173  }
2174 }
2175 
2176 #if !COMPILE_TEMPLATE_AMD3DNOW
2177 static void RENAME(extract_even2)(const uint8_t *src, uint8_t *dst0, uint8_t *dst1, x86_reg count)
2178 {
2179  dst0+= count;
2180  dst1+= count;
2181  src += 4*count;
2182  count= - count;
2183  if(count <= -8) {
2184  count += 7;
2185  __asm__ volatile(
2186  "pcmpeqw %%mm7, %%mm7 \n\t"
2187  "psrlw $8, %%mm7 \n\t"
2188  "1: \n\t"
2189  "movq -28(%1, %0, 4), %%mm0 \n\t"
2190  "movq -20(%1, %0, 4), %%mm1 \n\t"
2191  "movq -12(%1, %0, 4), %%mm2 \n\t"
2192  "movq -4(%1, %0, 4), %%mm3 \n\t"
2193  "pand %%mm7, %%mm0 \n\t"
2194  "pand %%mm7, %%mm1 \n\t"
2195  "pand %%mm7, %%mm2 \n\t"
2196  "pand %%mm7, %%mm3 \n\t"
2197  "packuswb %%mm1, %%mm0 \n\t"
2198  "packuswb %%mm3, %%mm2 \n\t"
2199  "movq %%mm0, %%mm1 \n\t"
2200  "movq %%mm2, %%mm3 \n\t"
2201  "psrlw $8, %%mm0 \n\t"
2202  "psrlw $8, %%mm2 \n\t"
2203  "pand %%mm7, %%mm1 \n\t"
2204  "pand %%mm7, %%mm3 \n\t"
2205  "packuswb %%mm2, %%mm0 \n\t"
2206  "packuswb %%mm3, %%mm1 \n\t"
2207  MOVNTQ" %%mm0,- 7(%3, %0) \n\t"
2208  MOVNTQ" %%mm1,- 7(%2, %0) \n\t"
2209  "add $8, %0 \n\t"
2210  " js 1b \n\t"
2211  : "+r"(count)
2212  : "r"(src), "r"(dst0), "r"(dst1)
2213  );
2214  count -= 7;
2215  }
2216  while(count<0) {
2217  dst0[count]= src[4*count+0];
2218  dst1[count]= src[4*count+2];
2219  count++;
2220  }
2221 }
2222 #endif /* !COMPILE_TEMPLATE_AMD3DNOW */
2223 
2224 static void RENAME(extract_even2avg)(const uint8_t *src0, const uint8_t *src1, uint8_t *dst0, uint8_t *dst1, x86_reg count)
2225 {
2226  dst0 += count;
2227  dst1 += count;
2228  src0 += 4*count;
2229  src1 += 4*count;
2230  count= - count;
2231 #ifdef PAVGB
2232  if(count <= -8) {
2233  count += 7;
2234  __asm__ volatile(
2235  "pcmpeqw %%mm7, %%mm7 \n\t"
2236  "psrlw $8, %%mm7 \n\t"
2237  "1: \n\t"
2238  "movq -28(%1, %0, 4), %%mm0 \n\t"
2239  "movq -20(%1, %0, 4), %%mm1 \n\t"
2240  "movq -12(%1, %0, 4), %%mm2 \n\t"
2241  "movq -4(%1, %0, 4), %%mm3 \n\t"
2242  PAVGB" -28(%2, %0, 4), %%mm0 \n\t"
2243  PAVGB" -20(%2, %0, 4), %%mm1 \n\t"
2244  PAVGB" -12(%2, %0, 4), %%mm2 \n\t"
2245  PAVGB" - 4(%2, %0, 4), %%mm3 \n\t"
2246  "pand %%mm7, %%mm0 \n\t"
2247  "pand %%mm7, %%mm1 \n\t"
2248  "pand %%mm7, %%mm2 \n\t"
2249  "pand %%mm7, %%mm3 \n\t"
2250  "packuswb %%mm1, %%mm0 \n\t"
2251  "packuswb %%mm3, %%mm2 \n\t"
2252  "movq %%mm0, %%mm1 \n\t"
2253  "movq %%mm2, %%mm3 \n\t"
2254  "psrlw $8, %%mm0 \n\t"
2255  "psrlw $8, %%mm2 \n\t"
2256  "pand %%mm7, %%mm1 \n\t"
2257  "pand %%mm7, %%mm3 \n\t"
2258  "packuswb %%mm2, %%mm0 \n\t"
2259  "packuswb %%mm3, %%mm1 \n\t"
2260  MOVNTQ" %%mm0,- 7(%4, %0) \n\t"
2261  MOVNTQ" %%mm1,- 7(%3, %0) \n\t"
2262  "add $8, %0 \n\t"
2263  " js 1b \n\t"
2264  : "+r"(count)
2265  : "r"(src0), "r"(src1), "r"(dst0), "r"(dst1)
2266  );
2267  count -= 7;
2268  }
2269 #endif
2270  while(count<0) {
2271  dst0[count]= (src0[4*count+0]+src1[4*count+0])>>1;
2272  dst1[count]= (src0[4*count+2]+src1[4*count+2])>>1;
2273  count++;
2274  }
2275 }
2276 
2277 #if !COMPILE_TEMPLATE_AMD3DNOW
2278 static void RENAME(extract_odd2)(const uint8_t *src, uint8_t *dst0, uint8_t *dst1, x86_reg count)
2279 {
2280  dst0+= count;
2281  dst1+= count;
2282  src += 4*count;
2283  count= - count;
2284  if(count <= -8) {
2285  count += 7;
2286  __asm__ volatile(
2287  "pcmpeqw %%mm7, %%mm7 \n\t"
2288  "psrlw $8, %%mm7 \n\t"
2289  "1: \n\t"
2290  "movq -28(%1, %0, 4), %%mm0 \n\t"
2291  "movq -20(%1, %0, 4), %%mm1 \n\t"
2292  "movq -12(%1, %0, 4), %%mm2 \n\t"
2293  "movq -4(%1, %0, 4), %%mm3 \n\t"
2294  "psrlw $8, %%mm0 \n\t"
2295  "psrlw $8, %%mm1 \n\t"
2296  "psrlw $8, %%mm2 \n\t"
2297  "psrlw $8, %%mm3 \n\t"
2298  "packuswb %%mm1, %%mm0 \n\t"
2299  "packuswb %%mm3, %%mm2 \n\t"
2300  "movq %%mm0, %%mm1 \n\t"
2301  "movq %%mm2, %%mm3 \n\t"
2302  "psrlw $8, %%mm0 \n\t"
2303  "psrlw $8, %%mm2 \n\t"
2304  "pand %%mm7, %%mm1 \n\t"
2305  "pand %%mm7, %%mm3 \n\t"
2306  "packuswb %%mm2, %%mm0 \n\t"
2307  "packuswb %%mm3, %%mm1 \n\t"
2308  MOVNTQ" %%mm0,- 7(%3, %0) \n\t"
2309  MOVNTQ" %%mm1,- 7(%2, %0) \n\t"
2310  "add $8, %0 \n\t"
2311  " js 1b \n\t"
2312  : "+r"(count)
2313  : "r"(src), "r"(dst0), "r"(dst1)
2314  );
2315  count -= 7;
2316  }
2317  src++;
2318  while(count<0) {
2319  dst0[count]= src[4*count+0];
2320  dst1[count]= src[4*count+2];
2321  count++;
2322  }
2323 }
2324 #endif /* !COMPILE_TEMPLATE_AMD3DNOW */
2325 
2326 static void RENAME(extract_odd2avg)(const uint8_t *src0, const uint8_t *src1, uint8_t *dst0, uint8_t *dst1, x86_reg count)
2327 {
2328  dst0 += count;
2329  dst1 += count;
2330  src0 += 4*count;
2331  src1 += 4*count;
2332  count= - count;
2333 #ifdef PAVGB
2334  if(count <= -8) {
2335  count += 7;
2336  __asm__ volatile(
2337  "pcmpeqw %%mm7, %%mm7 \n\t"
2338  "psrlw $8, %%mm7 \n\t"
2339  "1: \n\t"
2340  "movq -28(%1, %0, 4), %%mm0 \n\t"
2341  "movq -20(%1, %0, 4), %%mm1 \n\t"
2342  "movq -12(%1, %0, 4), %%mm2 \n\t"
2343  "movq -4(%1, %0, 4), %%mm3 \n\t"
2344  PAVGB" -28(%2, %0, 4), %%mm0 \n\t"
2345  PAVGB" -20(%2, %0, 4), %%mm1 \n\t"
2346  PAVGB" -12(%2, %0, 4), %%mm2 \n\t"
2347  PAVGB" - 4(%2, %0, 4), %%mm3 \n\t"
2348  "psrlw $8, %%mm0 \n\t"
2349  "psrlw $8, %%mm1 \n\t"
2350  "psrlw $8, %%mm2 \n\t"
2351  "psrlw $8, %%mm3 \n\t"
2352  "packuswb %%mm1, %%mm0 \n\t"
2353  "packuswb %%mm3, %%mm2 \n\t"
2354  "movq %%mm0, %%mm1 \n\t"
2355  "movq %%mm2, %%mm3 \n\t"
2356  "psrlw $8, %%mm0 \n\t"
2357  "psrlw $8, %%mm2 \n\t"
2358  "pand %%mm7, %%mm1 \n\t"
2359  "pand %%mm7, %%mm3 \n\t"
2360  "packuswb %%mm2, %%mm0 \n\t"
2361  "packuswb %%mm3, %%mm1 \n\t"
2362  MOVNTQ" %%mm0,- 7(%4, %0) \n\t"
2363  MOVNTQ" %%mm1,- 7(%3, %0) \n\t"
2364  "add $8, %0 \n\t"
2365  " js 1b \n\t"
2366  : "+r"(count)
2367  : "r"(src0), "r"(src1), "r"(dst0), "r"(dst1)
2368  );
2369  count -= 7;
2370  }
2371 #endif
2372  src0++;
2373  src1++;
2374  while(count<0) {
2375  dst0[count]= (src0[4*count+0]+src1[4*count+0])>>1;
2376  dst1[count]= (src0[4*count+2]+src1[4*count+2])>>1;
2377  count++;
2378  }
2379 }
2380 
2381 static void RENAME(yuyvtoyuv420)(uint8_t *ydst, uint8_t *udst, uint8_t *vdst, const uint8_t *src,
2382  int width, int height,
2383  int lumStride, int chromStride, int srcStride)
2384 {
2385  int y;
2386  const int chromWidth = FF_CEIL_RSHIFT(width, 1);
2387 
2388  for (y=0; y<height; y++) {
2389  RENAME(extract_even)(src, ydst, width);
2390  if(y&1) {
2391  RENAME(extract_odd2avg)(src-srcStride, src, udst, vdst, chromWidth);
2392  udst+= chromStride;
2393  vdst+= chromStride;
2394  }
2395 
2396  src += srcStride;
2397  ydst+= lumStride;
2398  }
2399  __asm__(
2400  EMMS" \n\t"
2401  SFENCE" \n\t"
2402  ::: "memory"
2403  );
2404 }
2405 
2406 #if !COMPILE_TEMPLATE_AMD3DNOW
2407 static void RENAME(yuyvtoyuv422)(uint8_t *ydst, uint8_t *udst, uint8_t *vdst, const uint8_t *src,
2408  int width, int height,
2409  int lumStride, int chromStride, int srcStride)
2410 {
2411  int y;
2412  const int chromWidth = FF_CEIL_RSHIFT(width, 1);
2413 
2414  for (y=0; y<height; y++) {
2415  RENAME(extract_even)(src, ydst, width);
2416  RENAME(extract_odd2)(src, udst, vdst, chromWidth);
2417 
2418  src += srcStride;
2419  ydst+= lumStride;
2420  udst+= chromStride;
2421  vdst+= chromStride;
2422  }
2423  __asm__(
2424  EMMS" \n\t"
2425  SFENCE" \n\t"
2426  ::: "memory"
2427  );
2428 }
2429 #endif /* !COMPILE_TEMPLATE_AMD3DNOW */
2430 
2431 static void RENAME(uyvytoyuv420)(uint8_t *ydst, uint8_t *udst, uint8_t *vdst, const uint8_t *src,
2432  int width, int height,
2433  int lumStride, int chromStride, int srcStride)
2434 {
2435  int y;
2436  const int chromWidth = FF_CEIL_RSHIFT(width, 1);
2437 
2438  for (y=0; y<height; y++) {
2439  RENAME(extract_even)(src+1, ydst, width);
2440  if(y&1) {
2441  RENAME(extract_even2avg)(src-srcStride, src, udst, vdst, chromWidth);
2442  udst+= chromStride;
2443  vdst+= chromStride;
2444  }
2445 
2446  src += srcStride;
2447  ydst+= lumStride;
2448  }
2449  __asm__(
2450  EMMS" \n\t"
2451  SFENCE" \n\t"
2452  ::: "memory"
2453  );
2454 }
2455 
2456 #if !COMPILE_TEMPLATE_AMD3DNOW
2457 static void RENAME(uyvytoyuv422)(uint8_t *ydst, uint8_t *udst, uint8_t *vdst, const uint8_t *src,
2458  int width, int height,
2459  int lumStride, int chromStride, int srcStride)
2460 {
2461  int y;
2462  const int chromWidth = FF_CEIL_RSHIFT(width, 1);
2463 
2464  for (y=0; y<height; y++) {
2465  RENAME(extract_even)(src+1, ydst, width);
2466  RENAME(extract_even2)(src, udst, vdst, chromWidth);
2467 
2468  src += srcStride;
2469  ydst+= lumStride;
2470  udst+= chromStride;
2471  vdst+= chromStride;
2472  }
2473  __asm__(
2474  EMMS" \n\t"
2475  SFENCE" \n\t"
2476  ::: "memory"
2477  );
2478 }
2479 #endif /* !COMPILE_TEMPLATE_AMD3DNOW */
2480 #endif /* !COMPILE_TEMPLATE_SSE2 */
2481 
2482 static av_cold void RENAME(rgb2rgb_init)(void)
2483 {
2484 #if !COMPILE_TEMPLATE_SSE2
2485 #if !COMPILE_TEMPLATE_AMD3DNOW
2486  rgb15to16 = RENAME(rgb15to16);
2487  rgb15tobgr24 = RENAME(rgb15tobgr24);
2488  rgb15to32 = RENAME(rgb15to32);
2489  rgb16tobgr24 = RENAME(rgb16tobgr24);
2490  rgb16to32 = RENAME(rgb16to32);
2491  rgb16to15 = RENAME(rgb16to15);
2492  rgb24tobgr16 = RENAME(rgb24tobgr16);
2493  rgb24tobgr15 = RENAME(rgb24tobgr15);
2494  rgb24tobgr32 = RENAME(rgb24tobgr32);
2495  rgb32to16 = RENAME(rgb32to16);
2496  rgb32to15 = RENAME(rgb32to15);
2497  rgb32tobgr24 = RENAME(rgb32tobgr24);
2498  rgb24to15 = RENAME(rgb24to15);
2499  rgb24to16 = RENAME(rgb24to16);
2500  rgb24tobgr24 = RENAME(rgb24tobgr24);
2501  shuffle_bytes_2103 = RENAME(shuffle_bytes_2103);
2502  rgb32tobgr16 = RENAME(rgb32tobgr16);
2503  rgb32tobgr15 = RENAME(rgb32tobgr15);
2504  yv12toyuy2 = RENAME(yv12toyuy2);
2505  yv12touyvy = RENAME(yv12touyvy);
2506  yuv422ptoyuy2 = RENAME(yuv422ptoyuy2);
2507  yuv422ptouyvy = RENAME(yuv422ptouyvy);
2508  yuy2toyv12 = RENAME(yuy2toyv12);
2509  vu9_to_vu12 = RENAME(vu9_to_vu12);
2510  yvu9_to_yuy2 = RENAME(yvu9_to_yuy2);
2511  uyvytoyuv422 = RENAME(uyvytoyuv422);
2512  yuyvtoyuv422 = RENAME(yuyvtoyuv422);
2513 #endif /* !COMPILE_TEMPLATE_AMD3DNOW */
2514 
2515 #if COMPILE_TEMPLATE_MMXEXT || COMPILE_TEMPLATE_AMD3DNOW
2516  planar2x = RENAME(planar2x);
2517 #endif /* COMPILE_TEMPLATE_MMXEXT || COMPILE_TEMPLATE_AMD3DNOW */
2518 #if HAVE_7REGS
2519  ff_rgb24toyv12 = RENAME(rgb24toyv12);
2520 #endif /* HAVE_7REGS */
2521 
2522  yuyvtoyuv420 = RENAME(yuyvtoyuv420);
2523  uyvytoyuv420 = RENAME(uyvytoyuv420);
2524 #endif /* !COMPILE_TEMPLATE_SSE2 */
2525 
2526 #if !COMPILE_TEMPLATE_AMD3DNOW && !COMPILE_TEMPLATE_AVX
2527  interleaveBytes = RENAME(interleaveBytes);
2528 #endif /* !COMPILE_TEMPLATE_AMD3DNOW && !COMPILE_TEMPLATE_AVX */
2529 #if !COMPILE_TEMPLATE_AVX || HAVE_AVX_EXTERNAL
2530 #if !COMPILE_TEMPLATE_AMD3DNOW && (ARCH_X86_32 || COMPILE_TEMPLATE_SSE2) && COMPILE_TEMPLATE_MMXEXT == COMPILE_TEMPLATE_SSE2 && HAVE_YASM
2531  deinterleaveBytes = RENAME(deinterleaveBytes);
2532 #endif
2533 #endif
2534 }
Generated on Sun Mar 23 2014 23:50:16 for FFmpeg by   doxygen 1.8.2