commit 18d72bf
sewn
·
2026-05-07 20:01:39 +0000 UTC
parent 96aabc6
import stb_image_resize2
2 files changed,
+10730,
-0
+10679,
-0
1@@ -0,0 +1,10679 @@
2+/* stb_image_resize2 - v2.18 - public domain image resizing
3+
4+ by Jeff Roberts (v2) and Jorge L Rodriguez
5+ http://github.com/nothings/stb
6+
7+ Can be threaded with the extended API. SSE2, AVX, Neon and WASM SIMD support. Only
8+ scaling and translation is supported, no rotations or shears.
9+
10+ COMPILING & LINKING
11+ In one C/C++ file that #includes this file, do this:
12+ #define STB_IMAGE_RESIZE_IMPLEMENTATION
13+ before the #include. That will create the implementation in that file.
14+
15+ EASY API CALLS:
16+ Easy API downsamples w/Mitchell filter, upsamples w/cubic interpolation, clamps to edge.
17+
18+ stbir_resize_uint8_srgb( input_pixels, input_w, input_h, input_stride_in_bytes,
19+ output_pixels, output_w, output_h, output_stride_in_bytes,
20+ pixel_layout_enum )
21+
22+ stbir_resize_uint8_linear( input_pixels, input_w, input_h, input_stride_in_bytes,
23+ output_pixels, output_w, output_h, output_stride_in_bytes,
24+ pixel_layout_enum )
25+
26+ stbir_resize_float_linear( input_pixels, input_w, input_h, input_stride_in_bytes,
27+ output_pixels, output_w, output_h, output_stride_in_bytes,
28+ pixel_layout_enum )
29+
30+ If you pass NULL or zero for the output_pixels, we will allocate the output buffer
31+ for you and return it from the function (free with free() or STBIR_FREE).
32+ As a special case, XX_stride_in_bytes of 0 means packed continuously in memory.
33+
34+ API LEVELS
35+ There are three levels of API - easy-to-use, medium-complexity and extended-complexity.
36+
37+ See the "header file" section of the source for API documentation.
38+
39+ ADDITIONAL DOCUMENTATION
40+
41+ MEMORY ALLOCATION
42+ By default, we use malloc and free for memory allocation. To override the
43+ memory allocation, before the implementation #include, add a:
44+
45+ #define STBIR_MALLOC(size,user_data) ...
46+ #define STBIR_FREE(ptr,user_data) ...
47+
48+ Each resize makes exactly one call to malloc/free (unless you use the
49+ extended API where you can do one allocation for many resizes). Under
50+ address sanitizer, we do separate allocations to find overread/writes.
51+
52+ PERFORMANCE
53+ This library was written with an emphasis on performance. When testing
54+ stb_image_resize with RGBA, the fastest mode is STBIR_4CHANNEL with
55+ STBIR_TYPE_UINT8 pixels and CLAMPed edges (which is what many other resize
56+ libs do by default). Also, make sure SIMD is turned on of course (default
57+ for 64-bit targets). Avoid WRAP edge mode if you want the fastest speed.
58+
59+ This library also comes with profiling built-in. If you define STBIR_PROFILE,
60+ you can use the advanced API and get low-level profiling information by
61+ calling stbir_resize_extended_profile_info() or stbir_resize_split_profile_info()
62+ after a resize.
63+
64+ SIMD
65+ Most of the routines have optimized SSE2, AVX, NEON and WASM versions.
66+
67+ On Microsoft compilers, we automatically turn on SIMD for 64-bit x64 and
68+ ARM; for 32-bit x86 and ARM, you select SIMD mode by defining STBIR_SSE2 or
69+ STBIR_NEON. For AVX and AVX2, we auto-select it by detecting the /arch:AVX
70+ or /arch:AVX2 switches. You can also always manually turn SSE2, AVX or AVX2
71+ support on by defining STBIR_SSE2, STBIR_AVX or STBIR_AVX2.
72+
73+ On Linux, SSE2 and Neon is on by default for 64-bit x64 or ARM64. For 32-bit,
74+ we select x86 SIMD mode by whether you have -msse2, -mavx or -mavx2 enabled
75+ on the command line. For 32-bit ARM, you must pass -mfpu=neon-vfpv4 for both
76+ clang and GCC, but GCC also requires an additional -mfp16-format=ieee to
77+ automatically enable NEON.
78+
79+ On x86 platforms, you can also define STBIR_FP16C to turn on FP16C instructions
80+ for converting back and forth to half-floats. This is autoselected when we
81+ are using AVX2. Clang and GCC also require the -mf16c switch. ARM always uses
82+ the built-in half float hardware NEON instructions.
83+
84+ You can also tell us to use multiply-add instructions with STBIR_USE_FMA.
85+ Because x86 doesn't always have fma, we turn it off by default to maintain
86+ determinism across all platforms. If you don't care about non-FMA determinism
87+ and are willing to restrict yourself to more recent x86 CPUs (around the AVX
88+ timeframe), then fma will give you around a 15% speedup.
89+
90+ You can force off SIMD in all cases by defining STBIR_NO_SIMD. You can turn
91+ off AVX or AVX2 specifically with STBIR_NO_AVX or STBIR_NO_AVX2. AVX is 10%
92+ to 40% faster, and AVX2 is generally another 12%.
93+
94+ ALPHA CHANNEL
95+ Most of the resizing functions provide the ability to control how the alpha
96+ channel of an image is processed.
97+
98+ When alpha represents transparency, it is important that when combining
99+ colors with filtering, the pixels should not be treated equally; they
100+ should use a weighted average based on their alpha values. For example,
101+ if a pixel is 1% opaque bright green and another pixel is 99% opaque
102+ black and you average them, the average will be 50% opaque, but the
103+ unweighted average and will be a middling green color, while the weighted
104+ average will be nearly black. This means the unweighted version introduced
105+ green energy that didn't exist in the source image.
106+
107+ (If you want to know why this makes sense, you can work out the math for
108+ the following: consider what happens if you alpha composite a source image
109+ over a fixed color and then average the output, vs. if you average the
110+ source image pixels and then composite that over the same fixed color.
111+ Only the weighted average produces the same result as the ground truth
112+ composite-then-average result.)
113+
114+ Therefore, it is in general best to "alpha weight" the pixels when applying
115+ filters to them. This essentially means multiplying the colors by the alpha
116+ values before combining them, and then dividing by the alpha value at the
117+ end.
118+
119+ The computer graphics industry introduced a technique called "premultiplied
120+ alpha" or "associated alpha" in which image colors are stored in image files
121+ already multiplied by their alpha. This saves some math when compositing,
122+ and also avoids the need to divide by the alpha at the end (which is quite
123+ inefficient). However, while premultiplied alpha is common in the movie CGI
124+ industry, it is not commonplace in other industries like videogames, and most
125+ consumer file formats are generally expected to contain not-premultiplied
126+ colors. For example, Photoshop saves PNG files "unpremultiplied", and web
127+ browsers like Chrome and Firefox expect PNG images to be unpremultiplied.
128+
129+ Note that there are three possibilities that might describe your image
130+ and resize expectation:
131+
132+ 1. images are not premultiplied, alpha weighting is desired
133+ 2. images are not premultiplied, alpha weighting is not desired
134+ 3. images are premultiplied
135+
136+ Both case #2 and case #3 require the exact same math: no alpha weighting
137+ should be applied or removed. Only case 1 requires extra math operations;
138+ the other two cases can be handled identically.
139+
140+ stb_image_resize expects case #1 by default, applying alpha weighting to
141+ images, expecting the input images to be unpremultiplied. This is what the
142+ COLOR+ALPHA buffer types tell the resizer to do.
143+
144+ When you use the pixel layouts STBIR_RGBA, STBIR_BGRA, STBIR_ARGB,
145+ STBIR_ABGR, STBIR_RA, or STBIR_AR you are telling us that the pixels are
146+ non-premultiplied. In these cases, the resizer will alpha weight the colors
147+ (effectively creating the premultiplied image), do the filtering, and then
148+ convert back to non-premult on exit.
149+
150+ When you use the pixel layouts STBIR_RGBA_PM, STBIR_BGRA_PM, STBIR_ARGB_PM,
151+ STBIR_ABGR_PM, STBIR_RA_PM or STBIR_AR_PM, you are telling that the pixels
152+ ARE premultiplied. In this case, the resizer doesn't have to do the
153+ premultipling - it can filter directly on the input. This about twice as
154+ fast as the non-premultiplied case, so it's the right option if your data is
155+ already setup correctly.
156+
157+ When you use the pixel layout STBIR_4CHANNEL or STBIR_2CHANNEL, you are
158+ telling us that there is no channel that represents transparency; it may be
159+ RGB and some unrelated fourth channel that has been stored in the alpha
160+ channel, but it is actually not alpha. No special processing will be
161+ performed.
162+
163+ The difference between the generic 4 or 2 channel layouts, and the
164+ specialized _PM versions is with the _PM versions you are telling us that
165+ the data *is* alpha, just don't premultiply it. That's important when
166+ using SRGB pixel formats, we need to know where the alpha is, because
167+ it is converted linearly (rather than with the SRGB converters).
168+
169+ Because alpha weighting produces the same effect as premultiplying, you
170+ even have the option with non-premultiplied inputs to let the resizer
171+ produce a premultiplied output. Because the intially computed alpha-weighted
172+ output image is effectively premultiplied, this is actually more performant
173+ than the normal path which un-premultiplies the output image as a final step.
174+
175+ Finally, when converting both in and out of non-premulitplied space (for
176+ example, when using STBIR_RGBA), we go to somewhat heroic measures to
177+ ensure that areas with zero alpha value pixels get something reasonable
178+ in the RGB values. If you don't care about the RGB values of zero alpha
179+ pixels, you can call the stbir_set_non_pm_alpha_speed_over_quality()
180+ function - this runs a premultiplied resize about 25% faster. That said,
181+ when you really care about speed, using premultiplied pixels for both in
182+ and out (STBIR_RGBA_PM, etc) much faster than both of these premultiplied
183+ options.
184+
185+ PIXEL LAYOUT CONVERSION
186+ The resizer can convert from some pixel layouts to others. When using the
187+ stbir_set_pixel_layouts(), you can, for example, specify STBIR_RGBA
188+ on input, and STBIR_ARGB on output, and it will re-organize the channels
189+ during the resize. Currently, you can only convert between two pixel
190+ layouts with the same number of channels.
191+
192+ DETERMINISM
193+ We commit to being deterministic (from x64 to ARM to scalar to SIMD, etc).
194+ This requires compiling with fast-math off (using at least /fp:precise).
195+ Also, you must turn off fp-contracting (which turns mult+adds into fmas)!
196+ We attempt to do this with pragmas, but with Clang, you usually want to add
197+ -ffp-contract=off to the command line as well.
198+
199+ For 32-bit x86, you must use SSE and SSE2 codegen for determinism. That is,
200+ if the scalar x87 unit gets used at all, we immediately lose determinism.
201+ On Microsoft Visual Studio 2008 and earlier, from what we can tell there is
202+ no way to be deterministic in 32-bit x86 (some x87 always leaks in, even
203+ with fp:strict). On 32-bit x86 GCC, determinism requires both -msse2 and
204+ -fpmath=sse.
205+
206+ Note that we will not be deterministic with float data containing NaNs -
207+ the NaNs will propagate differently on different SIMD and platforms.
208+
209+ If you turn on STBIR_USE_FMA, then we will be deterministic with other
210+ fma targets, but we will differ from non-fma targets (this is unavoidable,
211+ because a fma isn't simply an add with a mult - it also introduces a
212+ rounding difference compared to non-fma instruction sequences.
213+
214+ FLOAT PIXEL FORMAT RANGE
215+ Any range of values can be used for the non-alpha float data that you pass
216+ in (0 to 1, -1 to 1, whatever). However, if you are inputting float values
217+ but *outputting* bytes or shorts, you must use a range of 0 to 1 so that we
218+ scale back properly. The alpha channel must also be 0 to 1 for any format
219+ that does premultiplication prior to resizing.
220+
221+ Note also that with float output, using filters with negative lobes, the
222+ output filtered values might go slightly out of range. You can define
223+ STBIR_FLOAT_LOW_CLAMP and/or STBIR_FLOAT_HIGH_CLAMP to specify the range
224+ to clamp to on output, if that's important.
225+
226+ MAX/MIN SCALE FACTORS
227+ The input pixel resolutions are in integers, and we do the internal pointer
228+ resolution in size_t sized integers. However, the scale ratio from input
229+ resolution to output resolution is calculated in float form. This means
230+ the effective possible scale ratio is limited to 24 bits (or 16 million
231+ to 1). As you get close to the size of the float resolution (again, 16
232+ million pixels wide or high), you might start seeing float inaccuracy
233+ issues in general in the pipeline. If you have to do extreme resizes,
234+ you can usually do this is multiple stages (using float intermediate
235+ buffers).
236+
237+ FLIPPED IMAGES
238+ Stride is just the delta from one scanline to the next. This means you can
239+ use a negative stride to handle inverted images (point to the final
240+ scanline and use a negative stride). You can invert the input or output,
241+ using negative strides.
242+
243+ DEFAULT FILTERS
244+ For functions which don't provide explicit control over what filters to
245+ use, you can change the compile-time defaults with:
246+
247+ #define STBIR_DEFAULT_FILTER_UPSAMPLE STBIR_FILTER_something
248+ #define STBIR_DEFAULT_FILTER_DOWNSAMPLE STBIR_FILTER_something
249+
250+ See stbir_filter in the header-file section for the list of filters.
251+
252+ NEW FILTERS
253+ A number of 1D filter kernels are supplied. For a list of supported
254+ filters, see the stbir_filter enum. You can install your own filters by
255+ using the stbir_set_filter_callbacks function.
256+
257+ PROGRESS
258+ For interactive use with slow resize operations, you can use the
259+ scanline callbacks in the extended API. It would have to be a *very* large
260+ image resample to need progress though - we're very fast.
261+
262+ CEIL and FLOOR
263+ In scalar mode, the only functions we use from math.h are ceilf and floorf,
264+ but if you have your own versions, you can define the STBIR_CEILF(v) and
265+ STBIR_FLOORF(v) macros and we'll use them instead. In SIMD, we just use
266+ our own versions.
267+
268+ ASSERT
269+ Define STBIR_ASSERT(boolval) to override assert() and not use assert.h
270+
271+ PORTING FROM VERSION 1
272+ The API has changed. You can continue to use the old version of stb_image_resize.h,
273+ which is available in the "deprecated/" directory.
274+
275+ If you're using the old simple-to-use API, porting is straightforward.
276+ (For more advanced APIs, read the documentation.)
277+
278+ stbir_resize_uint8():
279+ - call `stbir_resize_uint8_linear`, cast channel count to `stbir_pixel_layout`
280+
281+ stbir_resize_float():
282+ - call `stbir_resize_float_linear`, cast channel count to `stbir_pixel_layout`
283+
284+ stbir_resize_uint8_srgb():
285+ - function name is unchanged
286+ - cast channel count to `stbir_pixel_layout`
287+ - above is sufficient unless your image has alpha and it's not RGBA/BGRA
288+ - in that case, follow the below instructions for stbir_resize_uint8_srgb_edgemode
289+
290+ stbir_resize_uint8_srgb_edgemode()
291+ - switch to the "medium complexity" API
292+ - stbir_resize(), very similar API but a few more parameters:
293+ - pixel_layout: cast channel count to `stbir_pixel_layout`
294+ - data_type: STBIR_TYPE_UINT8_SRGB
295+ - edge: unchanged (STBIR_EDGE_WRAP, etc.)
296+ - filter: STBIR_FILTER_DEFAULT
297+ - which channel is alpha is specified in stbir_pixel_layout, see enum for details
298+
299+ FUTURE TODOS
300+ * For polyphase integral filters, we just memcpy the coeffs to dupe
301+ them, but we should indirect and use the same coeff memory.
302+ * Add pixel layout conversions for sensible different channel counts
303+ (maybe, 1->3/4, 3->4, 4->1, 3->1).
304+ * For SIMD encode and decode scanline routines, do any pre-aligning
305+ for bad input/output buffer alignments and pitch?
306+ * For very wide scanlines, we should we do vertical strips to stay within
307+ L2 cache. Maybe do chunks of 1K pixels at a time. There would be
308+ some pixel reconversion, but probably dwarfed by things falling out
309+ of cache. Probably also something possible with alternating between
310+ scattering and gathering at high resize scales?
311+ * Should we have a multiple MIPs at the same time function (could keep
312+ more memory in cache during multiple resizes)?
313+ * Rewrite the coefficient generator to do many at once.
314+ * AVX-512 vertical kernels - worried about downclocking here.
315+ * Convert the reincludes to macros when we know they aren't changing.
316+ * Experiment with pivoting the horizontal and always using the
317+ vertical filters (which are faster, but perhaps not enough to overcome
318+ the pivot cost and the extra memory touches). Need to buffer the whole
319+ image so have to balance memory use.
320+ * Most of our code is internally function pointers, should we compile
321+ all the SIMD stuff always and dynamically dispatch?
322+
323+ CONTRIBUTORS
324+ Jeff Roberts: 2.0 implementation, optimizations, SIMD
325+ Martins Mozeiko: NEON simd, WASM simd, clang and GCC whisperer
326+ Fabian Giesen: half float and srgb converters
327+ Sean Barrett: API design, optimizations
328+ Jorge L Rodriguez: Original 1.0 implementation
329+ Aras Pranckevicius: bugfixes
330+ Nathan Reed: warning fixes for 1.0
331+
332+ REVISIONS
333+ 2.18 (2026-03-25) fixed coefficient calculation when skipping a coefficient off
334+ the left side of the window, added non-aligned access safe
335+ memcpy mode for scalar path, fixed various typos, and fixed
336+ define error in the float clamp output mode.
337+ 2.17 (2025-10-25) silly format bug in easy-to-use APIs.
338+ 2.16 (2025-10-21) fixed the easy-to-use APIs to allow inverted bitmaps (negative
339+ strides), fix vertical filter kernel callback, fix threaded
340+ gather buffer priming (and assert).
341+ (thanks adipose, TainZerL, and Harrison Green)
342+ 2.15 (2025-07-17) fixed an assert in debug mode when using floats with input
343+ callbacks, work around GCC warning when adding to null ptr
344+ (thanks Johannes Spohr and Pyry Kovanen).
345+ 2.14 (2025-05-09) fixed a bug using downsampling gather horizontal first, and
346+ scatter with vertical first.
347+ 2.13 (2025-02-27) fixed a bug when using input callbacks, turned off simd for
348+ tiny-c, fixed some variables that should have been static,
349+ fixes a bug when calculating temp memory with resizes that
350+ exceed 2GB of temp memory (very large resizes).
351+ 2.12 (2024-10-18) fix incorrect use of user_data with STBIR_FREE
352+ 2.11 (2024-09-08) fix harmless asan warnings in 2-channel and 3-channel mode
353+ with AVX-2, fix some weird scaling edge conditions with
354+ point sample mode.
355+ 2.10 (2024-07-27) fix the defines GCC and mingw for loop unroll control,
356+ fix MSVC 32-bit arm half float routines.
357+ 2.09 (2024-06-19) fix the defines for 32-bit ARM GCC builds (was selecting
358+ hardware half floats).
359+ 2.08 (2024-06-10) fix for RGB->BGR three channel flips and add SIMD (thanks
360+ to Ryan Salsbury), fix for sub-rect resizes, use the
361+ pragmas to control unrolling when they are available.
362+ 2.07 (2024-05-24) fix for slow final split during threaded conversions of very
363+ wide scanlines when downsampling (caused by extra input
364+ converting), fix for wide scanline resamples with many
365+ splits (int overflow), fix GCC warning.
366+ 2.06 (2024-02-10) fix for identical width/height 3x or more down-scaling
367+ undersampling a single row on rare resize ratios (about 1%).
368+ 2.05 (2024-02-07) fix for 2 pixel to 1 pixel resizes with wrap (thanks Aras),
369+ fix for output callback (thanks Julien Koenen).
370+ 2.04 (2023-11-17) fix for rare AVX bug, shadowed symbol (thanks Nikola Smiljanic).
371+ 2.03 (2023-11-01) ASAN and TSAN warnings fixed, minor tweaks.
372+ 2.00 (2023-10-10) mostly new source: new api, optimizations, simd, vertical-first, etc
373+ 2x-5x faster without simd, 4x-12x faster with simd,
374+ in some cases, 20x to 40x faster esp resizing large to very small.
375+ 0.96 (2019-03-04) fixed warnings
376+ 0.95 (2017-07-23) fixed warnings
377+ 0.94 (2017-03-18) fixed warnings
378+ 0.93 (2017-03-03) fixed bug with certain combinations of heights
379+ 0.92 (2017-01-02) fix integer overflow on large (>2GB) images
380+ 0.91 (2016-04-02) fix warnings; fix handling of subpixel regions
381+ 0.90 (2014-09-17) first released version
382+
383+ LICENSE
384+ See end of file for license information.
385+*/
386+
387+#if !defined(STB_IMAGE_RESIZE_DO_HORIZONTALS) && !defined(STB_IMAGE_RESIZE_DO_VERTICALS) && !defined(STB_IMAGE_RESIZE_DO_CODERS) // for internal re-includes
388+
389+#ifndef STBIR_INCLUDE_STB_IMAGE_RESIZE2_H
390+#define STBIR_INCLUDE_STB_IMAGE_RESIZE2_H
391+
392+#include <stddef.h>
393+#ifdef _MSC_VER
394+typedef unsigned char stbir_uint8;
395+typedef unsigned short stbir_uint16;
396+typedef unsigned int stbir_uint32;
397+typedef unsigned __int64 stbir_uint64;
398+#else
399+#include <stdint.h>
400+typedef uint8_t stbir_uint8;
401+typedef uint16_t stbir_uint16;
402+typedef uint32_t stbir_uint32;
403+typedef uint64_t stbir_uint64;
404+#endif
405+
406+#ifndef STBIRDEF
407+#ifdef STB_IMAGE_RESIZE_STATIC
408+#define STBIRDEF static
409+#else
410+#ifdef __cplusplus
411+#define STBIRDEF extern "C"
412+#else
413+#define STBIRDEF extern
414+#endif
415+#endif
416+#endif
417+
418+//////////////////////////////////////////////////////////////////////////////
419+//// start "header file" ///////////////////////////////////////////////////
420+//
421+// Easy-to-use API:
422+//
423+// * stride is the offset between successive rows of image data
424+// in memory, in bytes. specify 0 for packed continuously in memory
425+// * colorspace is linear or sRGB as specified by function name
426+// * Uses the default filters
427+// * Uses edge mode clamped
428+// * returned result is 1 for success or 0 in case of an error.
429+
430+
431+// stbir_pixel_layout specifies:
432+// number of channels
433+// order of channels
434+// whether color is premultiplied by alpha
435+// for back compatibility, you can cast the old channel count to an stbir_pixel_layout
436+typedef enum
437+{
438+ STBIR_1CHANNEL = 1,
439+ STBIR_2CHANNEL = 2,
440+ STBIR_RGB = 3, // 3-chan, with order specified (for channel flipping)
441+ STBIR_BGR = 0, // 3-chan, with order specified (for channel flipping)
442+ STBIR_4CHANNEL = 5,
443+
444+ STBIR_RGBA = 4, // alpha formats, where alpha is NOT premultiplied into color channels
445+ STBIR_BGRA = 6,
446+ STBIR_ARGB = 7,
447+ STBIR_ABGR = 8,
448+ STBIR_RA = 9,
449+ STBIR_AR = 10,
450+
451+ STBIR_RGBA_PM = 11, // alpha formats, where alpha is premultiplied into color channels
452+ STBIR_BGRA_PM = 12,
453+ STBIR_ARGB_PM = 13,
454+ STBIR_ABGR_PM = 14,
455+ STBIR_RA_PM = 15,
456+ STBIR_AR_PM = 16,
457+
458+ STBIR_RGBA_NO_AW = 11, // alpha formats, where NO alpha weighting is applied at all!
459+ STBIR_BGRA_NO_AW = 12, // these are just synonyms for the _PM flags (which also do
460+ STBIR_ARGB_NO_AW = 13, // no alpha weighting). These names just make it more clear
461+ STBIR_ABGR_NO_AW = 14, // for some folks).
462+ STBIR_RA_NO_AW = 15,
463+ STBIR_AR_NO_AW = 16,
464+
465+} stbir_pixel_layout;
466+
467+//===============================================================
468+// Simple-complexity API
469+//
470+// If output_pixels is NULL (0), then we will allocate the buffer and return it to you.
471+//--------------------------------
472+
473+STBIRDEF unsigned char * stbir_resize_uint8_srgb( const unsigned char *input_pixels , int input_w , int input_h, int input_stride_in_bytes,
474+ unsigned char *output_pixels, int output_w, int output_h, int output_stride_in_bytes,
475+ stbir_pixel_layout pixel_type );
476+
477+STBIRDEF unsigned char * stbir_resize_uint8_linear( const unsigned char *input_pixels , int input_w , int input_h, int input_stride_in_bytes,
478+ unsigned char *output_pixels, int output_w, int output_h, int output_stride_in_bytes,
479+ stbir_pixel_layout pixel_type );
480+
481+STBIRDEF float * stbir_resize_float_linear( const float *input_pixels , int input_w , int input_h, int input_stride_in_bytes,
482+ float *output_pixels, int output_w, int output_h, int output_stride_in_bytes,
483+ stbir_pixel_layout pixel_type );
484+//===============================================================
485+
486+//===============================================================
487+// Medium-complexity API
488+//
489+// This extends the easy-to-use API as follows:
490+//
491+// * Can specify the datatype - U8, U8_SRGB, U16, FLOAT, HALF_FLOAT
492+// * Edge wrap can selected explicitly
493+// * Filter can be selected explicitly
494+//--------------------------------
495+
496+typedef enum
497+{
498+ STBIR_EDGE_CLAMP = 0,
499+ STBIR_EDGE_REFLECT = 1,
500+ STBIR_EDGE_WRAP = 2, // this edge mode is slower and uses more memory
501+ STBIR_EDGE_ZERO = 3,
502+} stbir_edge;
503+
504+typedef enum
505+{
506+ STBIR_FILTER_DEFAULT = 0, // use same filter type that easy-to-use API chooses
507+ STBIR_FILTER_BOX = 1, // A trapezoid w/1-pixel wide ramps, same result as box for integer scale ratios
508+ STBIR_FILTER_TRIANGLE = 2, // On upsampling, produces same results as bilinear texture filtering
509+ STBIR_FILTER_CUBICBSPLINE = 3, // The cubic b-spline (aka Mitchell-Netrevalli with B=1,C=0), gaussian-esque
510+ STBIR_FILTER_CATMULLROM = 4, // An interpolating cubic spline
511+ STBIR_FILTER_MITCHELL = 5, // Mitchell-Netrevalli filter with B=1/3, C=1/3
512+ STBIR_FILTER_POINT_SAMPLE = 6, // Simple point sampling
513+ STBIR_FILTER_OTHER = 7, // User callback specified
514+} stbir_filter;
515+
516+typedef enum
517+{
518+ STBIR_TYPE_UINT8 = 0,
519+ STBIR_TYPE_UINT8_SRGB = 1,
520+ STBIR_TYPE_UINT8_SRGB_ALPHA = 2, // alpha channel, when present, should also be SRGB (this is very unusual)
521+ STBIR_TYPE_UINT16 = 3,
522+ STBIR_TYPE_FLOAT = 4,
523+ STBIR_TYPE_HALF_FLOAT = 5
524+} stbir_datatype;
525+
526+// medium api
527+STBIRDEF void * stbir_resize( const void *input_pixels , int input_w , int input_h, int input_stride_in_bytes,
528+ void *output_pixels, int output_w, int output_h, int output_stride_in_bytes,
529+ stbir_pixel_layout pixel_layout, stbir_datatype data_type,
530+ stbir_edge edge, stbir_filter filter );
531+//===============================================================
532+
533+
534+
535+//===============================================================
536+// Extended-complexity API
537+//
538+// This API exposes all resize functionality.
539+//
540+// * Separate filter types for each axis
541+// * Separate edge modes for each axis
542+// * Separate input and output data types
543+// * Can specify regions with subpixel correctness
544+// * Can specify alpha flags
545+// * Can specify a memory callback
546+// * Can specify a callback data type for pixel input and output
547+// * Can be threaded for a single resize
548+// * Can be used to resize many frames without recalculating the sampler info
549+//
550+// Use this API as follows:
551+// 1) Call the stbir_resize_init function on a local STBIR_RESIZE structure
552+// 2) Call any of the stbir_set functions
553+// 3) Optionally call stbir_build_samplers() if you are going to resample multiple times
554+// with the same input and output dimensions (like resizing video frames)
555+// 4) Resample by calling stbir_resize_extended().
556+// 5) Call stbir_free_samplers() if you called stbir_build_samplers()
557+//--------------------------------
558+
559+
560+// Types:
561+
562+// INPUT CALLBACK: this callback is used for input scanlines
563+typedef void const * stbir_input_callback( void * optional_output, void const * input_ptr, int num_pixels, int x, int y, void * context );
564+
565+// OUTPUT CALLBACK: this callback is used for output scanlines
566+typedef void stbir_output_callback( void const * output_ptr, int num_pixels, int y, void * context );
567+
568+// callbacks for user installed filters
569+typedef float stbir__kernel_callback( float x, float scale, void * user_data ); // centered at zero
570+typedef float stbir__support_callback( float scale, void * user_data );
571+
572+// internal structure with precomputed scaling
573+typedef struct stbir__info stbir__info;
574+
575+typedef struct STBIR_RESIZE // use the stbir_resize_init and stbir_override functions to set these values for future compatibility
576+{
577+ void * user_data;
578+ void const * input_pixels;
579+ int input_w, input_h;
580+ double input_s0, input_t0, input_s1, input_t1;
581+ stbir_input_callback * input_cb;
582+ void * output_pixels;
583+ int output_w, output_h;
584+ int output_subx, output_suby, output_subw, output_subh;
585+ stbir_output_callback * output_cb;
586+ int input_stride_in_bytes;
587+ int output_stride_in_bytes;
588+ int splits;
589+ int fast_alpha;
590+ int needs_rebuild;
591+ int called_alloc;
592+ stbir_pixel_layout input_pixel_layout_public;
593+ stbir_pixel_layout output_pixel_layout_public;
594+ stbir_datatype input_data_type;
595+ stbir_datatype output_data_type;
596+ stbir_filter horizontal_filter, vertical_filter;
597+ stbir_edge horizontal_edge, vertical_edge;
598+ stbir__kernel_callback * horizontal_filter_kernel; stbir__support_callback * horizontal_filter_support;
599+ stbir__kernel_callback * vertical_filter_kernel; stbir__support_callback * vertical_filter_support;
600+ stbir__info * samplers;
601+} STBIR_RESIZE;
602+
603+// extended complexity api
604+
605+
606+// First off, you must ALWAYS call stbir_resize_init on your resize structure before any of the other calls!
607+STBIRDEF void stbir_resize_init( STBIR_RESIZE * resize,
608+ const void *input_pixels, int input_w, int input_h, int input_stride_in_bytes, // stride can be zero
609+ void *output_pixels, int output_w, int output_h, int output_stride_in_bytes, // stride can be zero
610+ stbir_pixel_layout pixel_layout, stbir_datatype data_type );
611+
612+//===============================================================
613+// You can update these parameters any time after resize_init and there is no cost
614+//--------------------------------
615+
616+STBIRDEF void stbir_set_datatypes( STBIR_RESIZE * resize, stbir_datatype input_type, stbir_datatype output_type );
617+STBIRDEF void stbir_set_pixel_callbacks( STBIR_RESIZE * resize, stbir_input_callback * input_cb, stbir_output_callback * output_cb ); // no callbacks by default
618+STBIRDEF void stbir_set_user_data( STBIR_RESIZE * resize, void * user_data ); // pass back STBIR_RESIZE* by default
619+STBIRDEF void stbir_set_buffer_ptrs( STBIR_RESIZE * resize, const void * input_pixels, int input_stride_in_bytes, void * output_pixels, int output_stride_in_bytes );
620+
621+//===============================================================
622+
623+
624+//===============================================================
625+// If you call any of these functions, you will trigger a sampler rebuild!
626+//--------------------------------
627+
628+STBIRDEF int stbir_set_pixel_layouts( STBIR_RESIZE * resize, stbir_pixel_layout input_pixel_layout, stbir_pixel_layout output_pixel_layout ); // sets new buffer layouts
629+STBIRDEF int stbir_set_edgemodes( STBIR_RESIZE * resize, stbir_edge horizontal_edge, stbir_edge vertical_edge ); // CLAMP by default
630+
631+STBIRDEF int stbir_set_filters( STBIR_RESIZE * resize, stbir_filter horizontal_filter, stbir_filter vertical_filter ); // STBIR_DEFAULT_FILTER_UPSAMPLE/DOWNSAMPLE by default
632+STBIRDEF int stbir_set_filter_callbacks( STBIR_RESIZE * resize, stbir__kernel_callback * horizontal_filter, stbir__support_callback * horizontal_support, stbir__kernel_callback * vertical_filter, stbir__support_callback * vertical_support );
633+
634+STBIRDEF int stbir_set_pixel_subrect( STBIR_RESIZE * resize, int subx, int suby, int subw, int subh ); // sets both sub-regions (full regions by default)
635+STBIRDEF int stbir_set_input_subrect( STBIR_RESIZE * resize, double s0, double t0, double s1, double t1 ); // sets input sub-region (full region by default)
636+STBIRDEF int stbir_set_output_pixel_subrect( STBIR_RESIZE * resize, int subx, int suby, int subw, int subh ); // sets output sub-region (full region by default)
637+
638+// when inputting AND outputting non-premultiplied alpha pixels, we use a slower but higher quality technique
639+// that fills the zero alpha pixel's RGB values with something plausible. If you don't care about areas of
640+// zero alpha, you can call this function to get about a 25% speed improvement for STBIR_RGBA to STBIR_RGBA
641+// types of resizes.
642+STBIRDEF int stbir_set_non_pm_alpha_speed_over_quality( STBIR_RESIZE * resize, int non_pma_alpha_speed_over_quality );
643+//===============================================================
644+
645+
646+//===============================================================
647+// You can call build_samplers to prebuild all the internal data we need to resample.
648+// Then, if you call resize_extended many times with the same resize, you only pay the
649+// cost once.
650+// If you do call build_samplers, you MUST call free_samplers eventually.
651+//--------------------------------
652+
653+// This builds the samplers and does one allocation
654+STBIRDEF int stbir_build_samplers( STBIR_RESIZE * resize );
655+
656+// You MUST call this, if you call stbir_build_samplers or stbir_build_samplers_with_splits
657+STBIRDEF void stbir_free_samplers( STBIR_RESIZE * resize );
658+//===============================================================
659+
660+
661+// And this is the main function to perform the resize synchronously on one thread.
662+STBIRDEF int stbir_resize_extended( STBIR_RESIZE * resize );
663+
664+
665+//===============================================================
666+// Use these functions for multithreading.
667+// 1) You call stbir_build_samplers_with_splits first on the main thread
668+// 2) Then stbir_resize_with_split on each thread
669+// 3) stbir_free_samplers when done on the main thread
670+//--------------------------------
671+
672+// This will build samplers for threading.
673+// You can pass in the number of threads you'd like to use (try_splits).
674+// It returns the number of splits (threads) that you can call it with.
675+/// It might be less if the image resize can't be split up that many ways.
676+
677+STBIRDEF int stbir_build_samplers_with_splits( STBIR_RESIZE * resize, int try_splits );
678+
679+// This function does a split of the resizing (you call this fuction for each
680+// split, on multiple threads). A split is a piece of the output resize pixel space.
681+
682+// Note that you MUST call stbir_build_samplers_with_splits before stbir_resize_extended_split!
683+
684+// Usually, you will always call stbir_resize_split with split_start as the thread_index
685+// and "1" for the split_count.
686+// But, if you have a weird situation where you MIGHT want 8 threads, but sometimes
687+// only 4 threads, you can use 0,2,4,6 for the split_start's and use "2" for the
688+// split_count each time to turn in into a 4 thread resize. (This is unusual).
689+
690+STBIRDEF int stbir_resize_extended_split( STBIR_RESIZE * resize, int split_start, int split_count );
691+//===============================================================
692+
693+
694+//===============================================================
695+// Pixel Callbacks info:
696+//--------------------------------
697+
698+// The input callback is super flexible - it calls you with the input address
699+// (based on the stride and base pointer), it gives you an optional_output
700+// pointer that you can fill, or you can just return your own pointer into
701+// your own data.
702+//
703+// You can also do conversion from non-supported data types if necessary - in
704+// this case, you ignore the input_ptr and just use the x and y parameters to
705+// calculate your own input_ptr based on the size of each non-supported pixel.
706+// (Something like the third example below.)
707+//
708+// You can also install just an input or just an output callback by setting the
709+// callback that you don't want to zero.
710+//
711+// First example, progress: (getting a callback that you can monitor the progress):
712+// void const * my_callback( void * optional_output, void const * input_ptr, int num_pixels, int x, int y, void * context )
713+// {
714+// percentage_done = y / input_height;
715+// return input_ptr; // use buffer from call
716+// }
717+//
718+// Next example, copying: (copy from some other buffer or stream):
719+// void const * my_callback( void * optional_output, void const * input_ptr, int num_pixels, int x, int y, void * context )
720+// {
721+// CopyOrStreamData( optional_output, other_data_src, num_pixels * pixel_width_in_bytes );
722+// return optional_output; // return the optional buffer that we filled
723+// }
724+//
725+// Third example, input another buffer without copying: (zero-copy from other buffer):
726+// void const * my_callback( void * optional_output, void const * input_ptr, int num_pixels, int x, int y, void * context )
727+// {
728+// void * pixels = ( (char*) other_image_base ) + ( y * other_image_stride ) + ( x * other_pixel_width_in_bytes );
729+// return pixels; // return pointer to your data without copying
730+// }
731+//
732+//
733+// The output callback is considerably simpler - it just calls you so that you can dump
734+// out each scanline. You could even directly copy out to disk if you have a simple format
735+// like TGA or BMP. You can also convert to other output types here if you want.
736+//
737+// Simple example:
738+// void const * my_output( void * output_ptr, int num_pixels, int y, void * context )
739+// {
740+// percentage_done = y / output_height;
741+// fwrite( output_ptr, pixel_width_in_bytes, num_pixels, output_file );
742+// }
743+//===============================================================
744+
745+
746+
747+
748+//===============================================================
749+// optional built-in profiling API
750+//--------------------------------
751+
752+#ifdef STBIR_PROFILE
753+
754+typedef struct STBIR_PROFILE_INFO
755+{
756+ stbir_uint64 total_clocks;
757+
758+ // how many clocks spent (of total_clocks) in the various resize routines, along with a string description
759+ // there are "resize_count" number of zones
760+ stbir_uint64 clocks[ 8 ];
761+ char const ** descriptions;
762+
763+ // count of clocks and descriptions
764+ stbir_uint32 count;
765+} STBIR_PROFILE_INFO;
766+
767+// use after calling stbir_resize_extended (or stbir_build_samplers or stbir_build_samplers_with_splits)
768+STBIRDEF void stbir_resize_build_profile_info( STBIR_PROFILE_INFO * out_info, STBIR_RESIZE const * resize );
769+
770+// use after calling stbir_resize_extended
771+STBIRDEF void stbir_resize_extended_profile_info( STBIR_PROFILE_INFO * out_info, STBIR_RESIZE const * resize );
772+
773+// use after calling stbir_resize_extended_split
774+STBIRDEF void stbir_resize_split_profile_info( STBIR_PROFILE_INFO * out_info, STBIR_RESIZE const * resize, int split_start, int split_num );
775+
776+//===============================================================
777+
778+#endif
779+
780+
781+//// end header file /////////////////////////////////////////////////////
782+#endif // STBIR_INCLUDE_STB_IMAGE_RESIZE2_H
783+
784+#if defined(STB_IMAGE_RESIZE_IMPLEMENTATION) || defined(STB_IMAGE_RESIZE2_IMPLEMENTATION)
785+
786+#ifndef STBIR_ASSERT
787+#include <assert.h>
788+#define STBIR_ASSERT(x) assert(x)
789+#endif
790+
791+#ifndef STBIR_MALLOC
792+#include <stdlib.h>
793+#define STBIR_MALLOC(size,user_data) ((void)(user_data), malloc(size))
794+#define STBIR_FREE(ptr,user_data) ((void)(user_data), free(ptr))
795+// (we used the comma operator to evaluate user_data, to avoid "unused parameter" warnings)
796+#endif
797+
798+#ifdef _MSC_VER
799+
800+#define stbir__inline __forceinline
801+
802+#else
803+
804+#define stbir__inline __inline__
805+
806+// Clang address sanitizer
807+#if defined(__has_feature)
808+ #if __has_feature(address_sanitizer) || __has_feature(memory_sanitizer)
809+ #ifndef STBIR__SEPARATE_ALLOCATIONS
810+ #define STBIR__SEPARATE_ALLOCATIONS
811+ #endif
812+ #endif
813+#endif
814+
815+#endif
816+
817+// GCC and MSVC
818+#if defined(__SANITIZE_ADDRESS__)
819+ #ifndef STBIR__SEPARATE_ALLOCATIONS
820+ #define STBIR__SEPARATE_ALLOCATIONS
821+ #endif
822+#endif
823+
824+// Always turn off automatic FMA use - use STBIR_USE_FMA if you want.
825+// Otherwise, this is a determinism disaster.
826+#ifndef STBIR_DONT_CHANGE_FP_CONTRACT // override in case you don't want this behavior
827+#if defined(_MSC_VER) && !defined(__clang__)
828+#if _MSC_VER > 1200
829+#pragma fp_contract(off)
830+#endif
831+#elif defined(__GNUC__) && !defined(__clang__)
832+#pragma GCC optimize("fp-contract=off")
833+#else
834+#pragma STDC FP_CONTRACT OFF
835+#endif
836+#endif
837+
838+#ifdef _MSC_VER
839+#define STBIR__UNUSED(v) (void)(v)
840+#else
841+#define STBIR__UNUSED(v) (void)sizeof(v)
842+#endif
843+
844+#define STBIR__ARRAY_SIZE(a) (sizeof((a))/sizeof((a)[0]))
845+
846+
847+#ifndef STBIR_DEFAULT_FILTER_UPSAMPLE
848+#define STBIR_DEFAULT_FILTER_UPSAMPLE STBIR_FILTER_CATMULLROM
849+#endif
850+
851+#ifndef STBIR_DEFAULT_FILTER_DOWNSAMPLE
852+#define STBIR_DEFAULT_FILTER_DOWNSAMPLE STBIR_FILTER_MITCHELL
853+#endif
854+
855+
856+#ifndef STBIR__HEADER_FILENAME
857+#define STBIR__HEADER_FILENAME "stb_image_resize2.h"
858+#endif
859+
860+// the internal pixel layout enums are in a different order, so we can easily do range comparisons of types
861+// the public pixel layout is ordered in a way that if you cast num_channels (1-4) to the enum, you get something sensible
862+typedef enum
863+{
864+ STBIRI_1CHANNEL = 0,
865+ STBIRI_2CHANNEL = 1,
866+ STBIRI_RGB = 2,
867+ STBIRI_BGR = 3,
868+ STBIRI_4CHANNEL = 4,
869+
870+ STBIRI_RGBA = 5,
871+ STBIRI_BGRA = 6,
872+ STBIRI_ARGB = 7,
873+ STBIRI_ABGR = 8,
874+ STBIRI_RA = 9,
875+ STBIRI_AR = 10,
876+
877+ STBIRI_RGBA_PM = 11,
878+ STBIRI_BGRA_PM = 12,
879+ STBIRI_ARGB_PM = 13,
880+ STBIRI_ABGR_PM = 14,
881+ STBIRI_RA_PM = 15,
882+ STBIRI_AR_PM = 16,
883+} stbir_internal_pixel_layout;
884+
885+// define the public pixel layouts to not compile inside the implementation (to avoid accidental use)
886+#define STBIR_BGR bad_dont_use_in_implementation
887+#define STBIR_1CHANNEL STBIR_BGR
888+#define STBIR_2CHANNEL STBIR_BGR
889+#define STBIR_RGB STBIR_BGR
890+#define STBIR_RGBA STBIR_BGR
891+#define STBIR_4CHANNEL STBIR_BGR
892+#define STBIR_BGRA STBIR_BGR
893+#define STBIR_ARGB STBIR_BGR
894+#define STBIR_ABGR STBIR_BGR
895+#define STBIR_RA STBIR_BGR
896+#define STBIR_AR STBIR_BGR
897+#define STBIR_RGBA_PM STBIR_BGR
898+#define STBIR_BGRA_PM STBIR_BGR
899+#define STBIR_ARGB_PM STBIR_BGR
900+#define STBIR_ABGR_PM STBIR_BGR
901+#define STBIR_RA_PM STBIR_BGR
902+#define STBIR_AR_PM STBIR_BGR
903+
904+// must match stbir_datatype
905+static unsigned char stbir__type_size[] = {
906+ 1,1,1,2,4,2 // STBIR_TYPE_UINT8,STBIR_TYPE_UINT8_SRGB,STBIR_TYPE_UINT8_SRGB_ALPHA,STBIR_TYPE_UINT16,STBIR_TYPE_FLOAT,STBIR_TYPE_HALF_FLOAT
907+};
908+
909+// When gathering, the contributors are which source pixels contribute.
910+// When scattering, the contributors are which destination pixels are contributed to.
911+typedef struct
912+{
913+ int n0; // First contributing pixel
914+ int n1; // Last contributing pixel
915+} stbir__contributors;
916+
917+typedef struct
918+{
919+ int lowest; // First sample index for whole filter
920+ int highest; // Last sample index for whole filter
921+ int widest; // widest single set of samples for an output
922+} stbir__filter_extent_info;
923+
924+typedef struct
925+{
926+ int n0; // First pixel of decode buffer to write to
927+ int n1; // Last pixel of decode that will be written to
928+ int pixel_offset_for_input; // Pixel offset into input_scanline
929+} stbir__span;
930+
931+typedef struct stbir__scale_info
932+{
933+ int input_full_size;
934+ int output_sub_size;
935+ float scale;
936+ float inv_scale;
937+ float pixel_shift; // starting shift in output pixel space (in pixels)
938+ int scale_is_rational;
939+ stbir_uint32 scale_numerator, scale_denominator;
940+} stbir__scale_info;
941+
942+typedef struct
943+{
944+ stbir__contributors * contributors;
945+ float* coefficients;
946+ stbir__contributors * gather_prescatter_contributors;
947+ float * gather_prescatter_coefficients;
948+ stbir__scale_info scale_info;
949+ float support;
950+ stbir_filter filter_enum;
951+ stbir__kernel_callback * filter_kernel;
952+ stbir__support_callback * filter_support;
953+ stbir_edge edge;
954+ int coefficient_width;
955+ int filter_pixel_width;
956+ int filter_pixel_margin;
957+ int num_contributors;
958+ int contributors_size;
959+ int coefficients_size;
960+ stbir__filter_extent_info extent_info;
961+ int is_gather; // 0 = scatter, 1 = gather with scale >= 1, 2 = gather with scale < 1
962+ int gather_prescatter_num_contributors;
963+ int gather_prescatter_coefficient_width;
964+ int gather_prescatter_contributors_size;
965+ int gather_prescatter_coefficients_size;
966+} stbir__sampler;
967+
968+typedef struct
969+{
970+ stbir__contributors conservative;
971+ int edge_sizes[2]; // this can be less than filter_pixel_margin, if the filter and scaling falls off
972+ stbir__span spans[2]; // can be two spans, if doing input subrect with clamp mode WRAP
973+} stbir__extents;
974+
975+typedef struct
976+{
977+#ifdef STBIR_PROFILE
978+ union
979+ {
980+ struct { stbir_uint64 total, looping, vertical, horizontal, decode, encode, alpha, unalpha; } named;
981+ stbir_uint64 array[8];
982+ } profile;
983+ stbir_uint64 * current_zone_excluded_ptr;
984+#endif
985+ float* decode_buffer;
986+
987+ int ring_buffer_first_scanline;
988+ int ring_buffer_last_scanline;
989+ int ring_buffer_begin_index; // first_scanline is at this index in the ring buffer
990+ int start_output_y, end_output_y;
991+ int start_input_y, end_input_y; // used in scatter only
992+
993+ #ifdef STBIR__SEPARATE_ALLOCATIONS
994+ float** ring_buffers; // one pointer for each ring buffer
995+ #else
996+ float* ring_buffer; // one big buffer that we index into
997+ #endif
998+
999+ float* vertical_buffer;
1000+
1001+ char no_cache_straddle[64];
1002+} stbir__per_split_info;
1003+
1004+typedef float * stbir__decode_pixels_func( float * decode, int width_times_channels, void const * input );
1005+typedef void stbir__alpha_weight_func( float * decode_buffer, int width_times_channels );
1006+typedef void stbir__horizontal_gather_channels_func( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer,
1007+ stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width );
1008+typedef void stbir__alpha_unweight_func(float * encode_buffer, int width_times_channels );
1009+typedef void stbir__encode_pixels_func( void * output, int width_times_channels, float const * encode );
1010+
1011+struct stbir__info
1012+{
1013+#ifdef STBIR_PROFILE
1014+ union
1015+ {
1016+ struct { stbir_uint64 total, build, alloc, horizontal, vertical, cleanup, pivot; } named;
1017+ stbir_uint64 array[7];
1018+ } profile;
1019+ stbir_uint64 * current_zone_excluded_ptr;
1020+#endif
1021+ stbir__sampler horizontal;
1022+ stbir__sampler vertical;
1023+
1024+ void const * input_data;
1025+ void * output_data;
1026+
1027+ int input_stride_bytes;
1028+ int output_stride_bytes;
1029+ int ring_buffer_length_bytes; // The length of an individual entry in the ring buffer. The total number of ring buffers is stbir__get_filter_pixel_width(filter)
1030+ int ring_buffer_num_entries; // Total number of entries in the ring buffer.
1031+
1032+ stbir_datatype input_type;
1033+ stbir_datatype output_type;
1034+
1035+ stbir_input_callback * in_pixels_cb;
1036+ void * user_data;
1037+ stbir_output_callback * out_pixels_cb;
1038+
1039+ stbir__extents scanline_extents;
1040+
1041+ void * alloced_mem;
1042+ stbir__per_split_info * split_info; // by default 1, but there will be N of these allocated based on the thread init you did
1043+
1044+ stbir__decode_pixels_func * decode_pixels;
1045+ stbir__alpha_weight_func * alpha_weight;
1046+ stbir__horizontal_gather_channels_func * horizontal_gather_channels;
1047+ stbir__alpha_unweight_func * alpha_unweight;
1048+ stbir__encode_pixels_func * encode_pixels;
1049+
1050+ int alloc_ring_buffer_num_entries; // Number of entries in the ring buffer that will be allocated
1051+ int splits; // count of splits
1052+
1053+ stbir_internal_pixel_layout input_pixel_layout_internal;
1054+ stbir_internal_pixel_layout output_pixel_layout_internal;
1055+
1056+ int input_color_and_type;
1057+ int offset_x, offset_y; // offset within output_data
1058+ int vertical_first;
1059+ int channels;
1060+ int effective_channels; // same as channels, except on RGBA/ARGB (7), or XA/AX (3)
1061+ size_t alloced_total;
1062+};
1063+
1064+
1065+#define stbir__max_uint8_as_float 255.0f
1066+#define stbir__max_uint16_as_float 65535.0f
1067+#define stbir__max_uint8_as_float_inverted 3.9215689e-03f // (1.0f/255.0f)
1068+#define stbir__max_uint16_as_float_inverted 1.5259022e-05f // (1.0f/65535.0f)
1069+#define stbir__small_float ((float)1 / (1 << 20) / (1 << 20) / (1 << 20) / (1 << 20) / (1 << 20) / (1 << 20))
1070+
1071+// min/max friendly
1072+#define STBIR_CLAMP(x, xmin, xmax) for(;;) { \
1073+ if ( (x) < (xmin) ) (x) = (xmin); \
1074+ if ( (x) > (xmax) ) (x) = (xmax); \
1075+ break; \
1076+}
1077+
1078+static stbir__inline int stbir__min(int a, int b)
1079+{
1080+ return a < b ? a : b;
1081+}
1082+
1083+static stbir__inline int stbir__max(int a, int b)
1084+{
1085+ return a > b ? a : b;
1086+}
1087+
1088+static float stbir__srgb_uchar_to_linear_float[256] = {
1089+ 0.000000f, 0.000304f, 0.000607f, 0.000911f, 0.001214f, 0.001518f, 0.001821f, 0.002125f, 0.002428f, 0.002732f, 0.003035f,
1090+ 0.003347f, 0.003677f, 0.004025f, 0.004391f, 0.004777f, 0.005182f, 0.005605f, 0.006049f, 0.006512f, 0.006995f, 0.007499f,
1091+ 0.008023f, 0.008568f, 0.009134f, 0.009721f, 0.010330f, 0.010960f, 0.011612f, 0.012286f, 0.012983f, 0.013702f, 0.014444f,
1092+ 0.015209f, 0.015996f, 0.016807f, 0.017642f, 0.018500f, 0.019382f, 0.020289f, 0.021219f, 0.022174f, 0.023153f, 0.024158f,
1093+ 0.025187f, 0.026241f, 0.027321f, 0.028426f, 0.029557f, 0.030713f, 0.031896f, 0.033105f, 0.034340f, 0.035601f, 0.036889f,
1094+ 0.038204f, 0.039546f, 0.040915f, 0.042311f, 0.043735f, 0.045186f, 0.046665f, 0.048172f, 0.049707f, 0.051269f, 0.052861f,
1095+ 0.054480f, 0.056128f, 0.057805f, 0.059511f, 0.061246f, 0.063010f, 0.064803f, 0.066626f, 0.068478f, 0.070360f, 0.072272f,
1096+ 0.074214f, 0.076185f, 0.078187f, 0.080220f, 0.082283f, 0.084376f, 0.086500f, 0.088656f, 0.090842f, 0.093059f, 0.095307f,
1097+ 0.097587f, 0.099899f, 0.102242f, 0.104616f, 0.107023f, 0.109462f, 0.111932f, 0.114435f, 0.116971f, 0.119538f, 0.122139f,
1098+ 0.124772f, 0.127438f, 0.130136f, 0.132868f, 0.135633f, 0.138432f, 0.141263f, 0.144128f, 0.147027f, 0.149960f, 0.152926f,
1099+ 0.155926f, 0.158961f, 0.162029f, 0.165132f, 0.168269f, 0.171441f, 0.174647f, 0.177888f, 0.181164f, 0.184475f, 0.187821f,
1100+ 0.191202f, 0.194618f, 0.198069f, 0.201556f, 0.205079f, 0.208637f, 0.212231f, 0.215861f, 0.219526f, 0.223228f, 0.226966f,
1101+ 0.230740f, 0.234551f, 0.238398f, 0.242281f, 0.246201f, 0.250158f, 0.254152f, 0.258183f, 0.262251f, 0.266356f, 0.270498f,
1102+ 0.274677f, 0.278894f, 0.283149f, 0.287441f, 0.291771f, 0.296138f, 0.300544f, 0.304987f, 0.309469f, 0.313989f, 0.318547f,
1103+ 0.323143f, 0.327778f, 0.332452f, 0.337164f, 0.341914f, 0.346704f, 0.351533f, 0.356400f, 0.361307f, 0.366253f, 0.371238f,
1104+ 0.376262f, 0.381326f, 0.386430f, 0.391573f, 0.396755f, 0.401978f, 0.407240f, 0.412543f, 0.417885f, 0.423268f, 0.428691f,
1105+ 0.434154f, 0.439657f, 0.445201f, 0.450786f, 0.456411f, 0.462077f, 0.467784f, 0.473532f, 0.479320f, 0.485150f, 0.491021f,
1106+ 0.496933f, 0.502887f, 0.508881f, 0.514918f, 0.520996f, 0.527115f, 0.533276f, 0.539480f, 0.545725f, 0.552011f, 0.558340f,
1107+ 0.564712f, 0.571125f, 0.577581f, 0.584078f, 0.590619f, 0.597202f, 0.603827f, 0.610496f, 0.617207f, 0.623960f, 0.630757f,
1108+ 0.637597f, 0.644480f, 0.651406f, 0.658375f, 0.665387f, 0.672443f, 0.679543f, 0.686685f, 0.693872f, 0.701102f, 0.708376f,
1109+ 0.715694f, 0.723055f, 0.730461f, 0.737911f, 0.745404f, 0.752942f, 0.760525f, 0.768151f, 0.775822f, 0.783538f, 0.791298f,
1110+ 0.799103f, 0.806952f, 0.814847f, 0.822786f, 0.830770f, 0.838799f, 0.846873f, 0.854993f, 0.863157f, 0.871367f, 0.879622f,
1111+ 0.887923f, 0.896269f, 0.904661f, 0.913099f, 0.921582f, 0.930111f, 0.938686f, 0.947307f, 0.955974f, 0.964686f, 0.973445f,
1112+ 0.982251f, 0.991102f, 1.0f
1113+};
1114+
1115+typedef union
1116+{
1117+ unsigned int u;
1118+ float f;
1119+} stbir__FP32;
1120+
1121+// From https://gist.github.com/rygorous/2203834
1122+
1123+static const stbir_uint32 fp32_to_srgb8_tab4[104] = {
1124+ 0x0073000d, 0x007a000d, 0x0080000d, 0x0087000d, 0x008d000d, 0x0094000d, 0x009a000d, 0x00a1000d,
1125+ 0x00a7001a, 0x00b4001a, 0x00c1001a, 0x00ce001a, 0x00da001a, 0x00e7001a, 0x00f4001a, 0x0101001a,
1126+ 0x010e0033, 0x01280033, 0x01410033, 0x015b0033, 0x01750033, 0x018f0033, 0x01a80033, 0x01c20033,
1127+ 0x01dc0067, 0x020f0067, 0x02430067, 0x02760067, 0x02aa0067, 0x02dd0067, 0x03110067, 0x03440067,
1128+ 0x037800ce, 0x03df00ce, 0x044600ce, 0x04ad00ce, 0x051400ce, 0x057b00c5, 0x05dd00bc, 0x063b00b5,
1129+ 0x06970158, 0x07420142, 0x07e30130, 0x087b0120, 0x090b0112, 0x09940106, 0x0a1700fc, 0x0a9500f2,
1130+ 0x0b0f01cb, 0x0bf401ae, 0x0ccb0195, 0x0d950180, 0x0e56016e, 0x0f0d015e, 0x0fbc0150, 0x10630143,
1131+ 0x11070264, 0x1238023e, 0x1357021d, 0x14660201, 0x156601e9, 0x165a01d3, 0x174401c0, 0x182401af,
1132+ 0x18fe0331, 0x1a9602fe, 0x1c1502d2, 0x1d7e02ad, 0x1ed4028d, 0x201a0270, 0x21520256, 0x227d0240,
1133+ 0x239f0443, 0x25c003fe, 0x27bf03c4, 0x29a10392, 0x2b6a0367, 0x2d1d0341, 0x2ebe031f, 0x304d0300,
1134+ 0x31d105b0, 0x34a80555, 0x37520507, 0x39d504c5, 0x3c37048b, 0x3e7c0458, 0x40a8042a, 0x42bd0401,
1135+ 0x44c20798, 0x488e071e, 0x4c1c06b6, 0x4f76065d, 0x52a50610, 0x55ac05cc, 0x5892058f, 0x5b590559,
1136+ 0x5e0c0a23, 0x631c0980, 0x67db08f6, 0x6c55087f, 0x70940818, 0x74a007bd, 0x787d076c, 0x7c330723,
1137+};
1138+
1139+static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in)
1140+{
1141+ static const stbir__FP32 almostone = { 0x3f7fffff }; // 1-eps
1142+ static const stbir__FP32 minval = { (127-13) << 23 };
1143+ stbir_uint32 tab,bias,scale,t;
1144+ stbir__FP32 f;
1145+
1146+ // Clamp to [2^(-13), 1-eps]; these two values map to 0 and 1, respectively.
1147+ // The tests are carefully written so that NaNs map to 0, same as in the reference
1148+ // implementation.
1149+ if (!(in > minval.f)) // written this way to catch NaNs
1150+ return 0;
1151+ if (in > almostone.f)
1152+ return 255;
1153+
1154+ // Do the table lookup and unpack bias, scale
1155+ f.f = in;
1156+ tab = fp32_to_srgb8_tab4[(f.u - minval.u) >> 20];
1157+ bias = (tab >> 16) << 9;
1158+ scale = tab & 0xffff;
1159+
1160+ // Grab next-highest mantissa bits and perform linear interpolation
1161+ t = (f.u >> 12) & 0xff;
1162+ return (unsigned char) ((bias + scale*t) >> 16);
1163+}
1164+
1165+#ifndef STBIR_FORCE_GATHER_FILTER_SCANLINES_AMOUNT
1166+#define STBIR_FORCE_GATHER_FILTER_SCANLINES_AMOUNT 32 // when downsampling and <= 32 scanlines of buffering, use gather. gather used down to 1/8th scaling for 25% win.
1167+#endif
1168+
1169+#ifndef STBIR_FORCE_MINIMUM_SCANLINES_FOR_SPLITS
1170+#define STBIR_FORCE_MINIMUM_SCANLINES_FOR_SPLITS 4 // when threading, what is the minimum number of scanlines for a split?
1171+#endif
1172+
1173+#define STBIR_INPUT_CALLBACK_PADDING 3
1174+
1175+#ifdef _M_IX86_FP
1176+#if ( _M_IX86_FP >= 1 )
1177+#ifndef STBIR_SSE
1178+#define STBIR_SSE
1179+#endif
1180+#endif
1181+#endif
1182+
1183+#ifdef __TINYC__
1184+ // tiny c has no intrinsics yet - this can become a version check if they add them
1185+ #define STBIR_NO_SIMD
1186+#endif
1187+
1188+#if defined(_x86_64) || defined( __x86_64__ ) || defined( _M_X64 ) || defined(__x86_64) || defined(_M_AMD64) || defined(__SSE2__) || defined(STBIR_SSE) || defined(STBIR_SSE2)
1189+ #ifndef STBIR_SSE2
1190+ #define STBIR_SSE2
1191+ #endif
1192+ #if defined(__AVX__) || defined(STBIR_AVX2)
1193+ #ifndef STBIR_AVX
1194+ #ifndef STBIR_NO_AVX
1195+ #define STBIR_AVX
1196+ #endif
1197+ #endif
1198+ #endif
1199+ #if defined(__AVX2__) || defined(STBIR_AVX2)
1200+ #ifndef STBIR_NO_AVX2
1201+ #ifndef STBIR_AVX2
1202+ #define STBIR_AVX2
1203+ #endif
1204+ #if defined( _MSC_VER ) && !defined(__clang__)
1205+ #ifndef STBIR_FP16C // FP16C instructions are on all AVX2 cpus, so we can autoselect it here on microsoft - clang needs -mf16c
1206+ #define STBIR_FP16C
1207+ #endif
1208+ #endif
1209+ #endif
1210+ #endif
1211+ #ifdef __F16C__
1212+ #ifndef STBIR_FP16C // turn on FP16C instructions if the define is set (for clang and gcc)
1213+ #define STBIR_FP16C
1214+ #endif
1215+ #endif
1216+#endif
1217+
1218+#if defined( _M_ARM64 ) || defined( __aarch64__ ) || defined( __arm64__ ) || ((__ARM_NEON_FP & 4) != 0) || defined(__ARM_NEON__)
1219+#ifndef STBIR_NEON
1220+#define STBIR_NEON
1221+#endif
1222+#endif
1223+
1224+#if defined(_M_ARM) || defined(__arm__)
1225+#ifdef STBIR_USE_FMA
1226+#undef STBIR_USE_FMA // no FMA for 32-bit arm on MSVC
1227+#endif
1228+#endif
1229+
1230+#if defined(__wasm__) && defined(__wasm_simd128__)
1231+#ifndef STBIR_WASM
1232+#define STBIR_WASM
1233+#endif
1234+#endif
1235+
1236+// restrict pointers for the output pointers, other loop and unroll control
1237+#if defined( _MSC_VER ) && !defined(__clang__)
1238+ #define STBIR_STREAMOUT_PTR( star ) star __restrict
1239+ #define STBIR_NO_UNROLL( ptr ) __assume(ptr) // this oddly keeps msvc from unrolling a loop
1240+ #if _MSC_VER >= 1900
1241+ #define STBIR_NO_UNROLL_LOOP_START __pragma(loop( no_vector ))
1242+ #else
1243+ #define STBIR_NO_UNROLL_LOOP_START
1244+ #endif
1245+#elif defined( __clang__ )
1246+ #define STBIR_STREAMOUT_PTR( star ) star __restrict__
1247+ #define STBIR_NO_UNROLL( ptr ) __asm__ (""::"r"(ptr))
1248+ #if ( __clang_major__ >= 4 ) || ( ( __clang_major__ >= 3 ) && ( __clang_minor__ >= 5 ) )
1249+ #define STBIR_NO_UNROLL_LOOP_START _Pragma("clang loop unroll(disable)") _Pragma("clang loop vectorize(disable)")
1250+ #else
1251+ #define STBIR_NO_UNROLL_LOOP_START
1252+ #endif
1253+#elif defined( __GNUC__ )
1254+ #define STBIR_STREAMOUT_PTR( star ) star __restrict__
1255+ #define STBIR_NO_UNROLL( ptr ) __asm__ (""::"r"(ptr))
1256+ #if __GNUC__ >= 14
1257+ #define STBIR_NO_UNROLL_LOOP_START _Pragma("GCC unroll 0") _Pragma("GCC novector")
1258+ #else
1259+ #define STBIR_NO_UNROLL_LOOP_START
1260+ #endif
1261+ #define STBIR_NO_UNROLL_LOOP_START_INF_FOR
1262+#else
1263+ #define STBIR_STREAMOUT_PTR( star ) star
1264+ #define STBIR_NO_UNROLL( ptr )
1265+ #define STBIR_NO_UNROLL_LOOP_START
1266+#endif
1267+
1268+#ifndef STBIR_NO_UNROLL_LOOP_START_INF_FOR
1269+#define STBIR_NO_UNROLL_LOOP_START_INF_FOR STBIR_NO_UNROLL_LOOP_START
1270+#endif
1271+
1272+#ifdef STBIR_NO_SIMD // force simd off for whatever reason
1273+
1274+// force simd off overrides everything else, so clear it all
1275+
1276+#ifdef STBIR_SSE2
1277+#undef STBIR_SSE2
1278+#endif
1279+
1280+#ifdef STBIR_AVX
1281+#undef STBIR_AVX
1282+#endif
1283+
1284+#ifdef STBIR_NEON
1285+#undef STBIR_NEON
1286+#endif
1287+
1288+#ifdef STBIR_AVX2
1289+#undef STBIR_AVX2
1290+#endif
1291+
1292+#ifdef STBIR_FP16C
1293+#undef STBIR_FP16C
1294+#endif
1295+
1296+#ifdef STBIR_WASM
1297+#undef STBIR_WASM
1298+#endif
1299+
1300+#ifdef STBIR_SIMD
1301+#undef STBIR_SIMD
1302+#endif
1303+
1304+#else // STBIR_SIMD
1305+
1306+#ifdef STBIR_SSE2
1307+ #include <emmintrin.h>
1308+
1309+ #define stbir__simdf __m128
1310+ #define stbir__simdi __m128i
1311+
1312+ #define stbir_simdi_castf( reg ) _mm_castps_si128(reg)
1313+ #define stbir_simdf_casti( reg ) _mm_castsi128_ps(reg)
1314+
1315+ #define stbir__simdf_load( reg, ptr ) (reg) = _mm_loadu_ps( (float const*)(ptr) )
1316+ #define stbir__simdi_load( reg, ptr ) (reg) = _mm_loadu_si128 ( (stbir__simdi const*)(ptr) )
1317+ #define stbir__simdf_load1( out, ptr ) (out) = _mm_load_ss( (float const*)(ptr) ) // top values can be random (not denormal or nan for perf)
1318+ #define stbir__simdi_load1( out, ptr ) (out) = _mm_castps_si128( _mm_load_ss( (float const*)(ptr) ))
1319+ #define stbir__simdf_load1z( out, ptr ) (out) = _mm_load_ss( (float const*)(ptr) ) // top values must be zero
1320+ #define stbir__simdf_frep4( fvar ) _mm_set_ps1( fvar )
1321+ #define stbir__simdf_load1frep4( out, fvar ) (out) = _mm_set_ps1( fvar )
1322+ #define stbir__simdf_load2( out, ptr ) (out) = _mm_castsi128_ps( _mm_loadl_epi64( (__m128i*)(ptr)) ) // top values can be random (not denormal or nan for perf)
1323+ #define stbir__simdf_load2z( out, ptr ) (out) = _mm_castsi128_ps( _mm_loadl_epi64( (__m128i*)(ptr)) ) // top values must be zero
1324+ #define stbir__simdf_load2hmerge( out, reg, ptr ) (out) = _mm_castpd_ps(_mm_loadh_pd( _mm_castps_pd(reg), (double*)(ptr) ))
1325+
1326+ #define stbir__simdf_zeroP() _mm_setzero_ps()
1327+ #define stbir__simdf_zero( reg ) (reg) = _mm_setzero_ps()
1328+
1329+ #define stbir__simdf_store( ptr, reg ) _mm_storeu_ps( (float*)(ptr), reg )
1330+ #define stbir__simdf_store1( ptr, reg ) _mm_store_ss( (float*)(ptr), reg )
1331+ #define stbir__simdf_store2( ptr, reg ) _mm_storel_epi64( (__m128i*)(ptr), _mm_castps_si128(reg) )
1332+ #define stbir__simdf_store2h( ptr, reg ) _mm_storeh_pd( (double*)(ptr), _mm_castps_pd(reg) )
1333+
1334+ #define stbir__simdi_store( ptr, reg ) _mm_storeu_si128( (__m128i*)(ptr), reg )
1335+ #define stbir__simdi_store1( ptr, reg ) _mm_store_ss( (float*)(ptr), _mm_castsi128_ps(reg) )
1336+ #define stbir__simdi_store2( ptr, reg ) _mm_storel_epi64( (__m128i*)(ptr), (reg) )
1337+
1338+ #define stbir__prefetch( ptr ) _mm_prefetch((char*)(ptr), _MM_HINT_T0 )
1339+
1340+ #define stbir__simdi_expand_u8_to_u32(out0,out1,out2,out3,ireg) \
1341+ { \
1342+ stbir__simdi zero = _mm_setzero_si128(); \
1343+ out2 = _mm_unpacklo_epi8( ireg, zero ); \
1344+ out3 = _mm_unpackhi_epi8( ireg, zero ); \
1345+ out0 = _mm_unpacklo_epi16( out2, zero ); \
1346+ out1 = _mm_unpackhi_epi16( out2, zero ); \
1347+ out2 = _mm_unpacklo_epi16( out3, zero ); \
1348+ out3 = _mm_unpackhi_epi16( out3, zero ); \
1349+ }
1350+
1351+#define stbir__simdi_expand_u8_to_1u32(out,ireg) \
1352+ { \
1353+ stbir__simdi zero = _mm_setzero_si128(); \
1354+ out = _mm_unpacklo_epi8( ireg, zero ); \
1355+ out = _mm_unpacklo_epi16( out, zero ); \
1356+ }
1357+
1358+ #define stbir__simdi_expand_u16_to_u32(out0,out1,ireg) \
1359+ { \
1360+ stbir__simdi zero = _mm_setzero_si128(); \
1361+ out0 = _mm_unpacklo_epi16( ireg, zero ); \
1362+ out1 = _mm_unpackhi_epi16( ireg, zero ); \
1363+ }
1364+
1365+ #define stbir__simdf_convert_float_to_i32( i, f ) (i) = _mm_cvttps_epi32(f)
1366+ #define stbir__simdf_convert_float_to_int( f ) _mm_cvtt_ss2si(f)
1367+ #define stbir__simdf_convert_float_to_uint8( f ) ((unsigned char)_mm_cvtsi128_si32(_mm_cvttps_epi32(_mm_max_ps(_mm_min_ps(f,STBIR__CONSTF(STBIR_max_uint8_as_float)),_mm_setzero_ps()))))
1368+ #define stbir__simdf_convert_float_to_short( f ) ((unsigned short)_mm_cvtsi128_si32(_mm_cvttps_epi32(_mm_max_ps(_mm_min_ps(f,STBIR__CONSTF(STBIR_max_uint16_as_float)),_mm_setzero_ps()))))
1369+
1370+ #define stbir__simdi_to_int( i ) _mm_cvtsi128_si32(i)
1371+ #define stbir__simdi_convert_i32_to_float(out, ireg) (out) = _mm_cvtepi32_ps( ireg )
1372+ #define stbir__simdf_add( out, reg0, reg1 ) (out) = _mm_add_ps( reg0, reg1 )
1373+ #define stbir__simdf_mult( out, reg0, reg1 ) (out) = _mm_mul_ps( reg0, reg1 )
1374+ #define stbir__simdf_mult_mem( out, reg, ptr ) (out) = _mm_mul_ps( reg, _mm_loadu_ps( (float const*)(ptr) ) )
1375+ #define stbir__simdf_mult1_mem( out, reg, ptr ) (out) = _mm_mul_ss( reg, _mm_load_ss( (float const*)(ptr) ) )
1376+ #define stbir__simdf_add_mem( out, reg, ptr ) (out) = _mm_add_ps( reg, _mm_loadu_ps( (float const*)(ptr) ) )
1377+ #define stbir__simdf_add1_mem( out, reg, ptr ) (out) = _mm_add_ss( reg, _mm_load_ss( (float const*)(ptr) ) )
1378+
1379+ #ifdef STBIR_USE_FMA // not on by default to maintain bit identical simd to non-simd
1380+ #include <immintrin.h>
1381+ #define stbir__simdf_madd( out, add, mul1, mul2 ) (out) = _mm_fmadd_ps( mul1, mul2, add )
1382+ #define stbir__simdf_madd1( out, add, mul1, mul2 ) (out) = _mm_fmadd_ss( mul1, mul2, add )
1383+ #define stbir__simdf_madd_mem( out, add, mul, ptr ) (out) = _mm_fmadd_ps( mul, _mm_loadu_ps( (float const*)(ptr) ), add )
1384+ #define stbir__simdf_madd1_mem( out, add, mul, ptr ) (out) = _mm_fmadd_ss( mul, _mm_load_ss( (float const*)(ptr) ), add )
1385+ #else
1386+ #define stbir__simdf_madd( out, add, mul1, mul2 ) (out) = _mm_add_ps( add, _mm_mul_ps( mul1, mul2 ) )
1387+ #define stbir__simdf_madd1( out, add, mul1, mul2 ) (out) = _mm_add_ss( add, _mm_mul_ss( mul1, mul2 ) )
1388+ #define stbir__simdf_madd_mem( out, add, mul, ptr ) (out) = _mm_add_ps( add, _mm_mul_ps( mul, _mm_loadu_ps( (float const*)(ptr) ) ) )
1389+ #define stbir__simdf_madd1_mem( out, add, mul, ptr ) (out) = _mm_add_ss( add, _mm_mul_ss( mul, _mm_load_ss( (float const*)(ptr) ) ) )
1390+ #endif
1391+
1392+ #define stbir__simdf_add1( out, reg0, reg1 ) (out) = _mm_add_ss( reg0, reg1 )
1393+ #define stbir__simdf_mult1( out, reg0, reg1 ) (out) = _mm_mul_ss( reg0, reg1 )
1394+
1395+ #define stbir__simdf_and( out, reg0, reg1 ) (out) = _mm_and_ps( reg0, reg1 )
1396+ #define stbir__simdf_or( out, reg0, reg1 ) (out) = _mm_or_ps( reg0, reg1 )
1397+
1398+ #define stbir__simdf_min( out, reg0, reg1 ) (out) = _mm_min_ps( reg0, reg1 )
1399+ #define stbir__simdf_max( out, reg0, reg1 ) (out) = _mm_max_ps( reg0, reg1 )
1400+ #define stbir__simdf_min1( out, reg0, reg1 ) (out) = _mm_min_ss( reg0, reg1 )
1401+ #define stbir__simdf_max1( out, reg0, reg1 ) (out) = _mm_max_ss( reg0, reg1 )
1402+
1403+ #define stbir__simdf_0123ABCDto3ABx( out, reg0, reg1 ) (out)=_mm_castsi128_ps( _mm_shuffle_epi32( _mm_castps_si128( _mm_shuffle_ps( reg1,reg0, (0<<0) + (1<<2) + (2<<4) + (3<<6) )), (3<<0) + (0<<2) + (1<<4) + (2<<6) ) )
1404+ #define stbir__simdf_0123ABCDto23Ax( out, reg0, reg1 ) (out)=_mm_castsi128_ps( _mm_shuffle_epi32( _mm_castps_si128( _mm_shuffle_ps( reg1,reg0, (0<<0) + (1<<2) + (2<<4) + (3<<6) )), (2<<0) + (3<<2) + (0<<4) + (1<<6) ) )
1405+
1406+ static const stbir__simdf STBIR_zeroones = { 0.0f,1.0f,0.0f,1.0f };
1407+ static const stbir__simdf STBIR_onezeros = { 1.0f,0.0f,1.0f,0.0f };
1408+ #define stbir__simdf_aaa1( out, alp, ones ) (out)=_mm_castsi128_ps( _mm_shuffle_epi32( _mm_castps_si128( _mm_movehl_ps( ones, alp ) ), (1<<0) + (1<<2) + (1<<4) + (2<<6) ) )
1409+ #define stbir__simdf_1aaa( out, alp, ones ) (out)=_mm_castsi128_ps( _mm_shuffle_epi32( _mm_castps_si128( _mm_movelh_ps( ones, alp ) ), (0<<0) + (2<<2) + (2<<4) + (2<<6) ) )
1410+ #define stbir__simdf_a1a1( out, alp, ones) (out) = _mm_or_ps( _mm_castsi128_ps( _mm_srli_epi64( _mm_castps_si128(alp), 32 ) ), STBIR_zeroones )
1411+ #define stbir__simdf_1a1a( out, alp, ones) (out) = _mm_or_ps( _mm_castsi128_ps( _mm_slli_epi64( _mm_castps_si128(alp), 32 ) ), STBIR_onezeros )
1412+
1413+ #define stbir__simdf_swiz( reg, one, two, three, four ) _mm_castsi128_ps( _mm_shuffle_epi32( _mm_castps_si128( reg ), (one<<0) + (two<<2) + (three<<4) + (four<<6) ) )
1414+
1415+ #define stbir__simdi_and( out, reg0, reg1 ) (out) = _mm_and_si128( reg0, reg1 )
1416+ #define stbir__simdi_or( out, reg0, reg1 ) (out) = _mm_or_si128( reg0, reg1 )
1417+ #define stbir__simdi_16madd( out, reg0, reg1 ) (out) = _mm_madd_epi16( reg0, reg1 )
1418+
1419+ #define stbir__simdf_pack_to_8bytes(out,aa,bb) \
1420+ { \
1421+ stbir__simdf af,bf; \
1422+ stbir__simdi a,b; \
1423+ af = _mm_min_ps( aa, STBIR_max_uint8_as_float ); \
1424+ bf = _mm_min_ps( bb, STBIR_max_uint8_as_float ); \
1425+ af = _mm_max_ps( af, _mm_setzero_ps() ); \
1426+ bf = _mm_max_ps( bf, _mm_setzero_ps() ); \
1427+ a = _mm_cvttps_epi32( af ); \
1428+ b = _mm_cvttps_epi32( bf ); \
1429+ a = _mm_packs_epi32( a, b ); \
1430+ out = _mm_packus_epi16( a, a ); \
1431+ }
1432+
1433+ #define stbir__simdf_load4_transposed( o0, o1, o2, o3, ptr ) \
1434+ stbir__simdf_load( o0, (ptr) ); \
1435+ stbir__simdf_load( o1, (ptr)+4 ); \
1436+ stbir__simdf_load( o2, (ptr)+8 ); \
1437+ stbir__simdf_load( o3, (ptr)+12 ); \
1438+ { \
1439+ __m128 tmp0, tmp1, tmp2, tmp3; \
1440+ tmp0 = _mm_unpacklo_ps(o0, o1); \
1441+ tmp2 = _mm_unpacklo_ps(o2, o3); \
1442+ tmp1 = _mm_unpackhi_ps(o0, o1); \
1443+ tmp3 = _mm_unpackhi_ps(o2, o3); \
1444+ o0 = _mm_movelh_ps(tmp0, tmp2); \
1445+ o1 = _mm_movehl_ps(tmp2, tmp0); \
1446+ o2 = _mm_movelh_ps(tmp1, tmp3); \
1447+ o3 = _mm_movehl_ps(tmp3, tmp1); \
1448+ }
1449+
1450+ #define stbir__interleave_pack_and_store_16_u8( ptr, r0, r1, r2, r3 ) \
1451+ r0 = _mm_packs_epi32( r0, r1 ); \
1452+ r2 = _mm_packs_epi32( r2, r3 ); \
1453+ r1 = _mm_unpacklo_epi16( r0, r2 ); \
1454+ r3 = _mm_unpackhi_epi16( r0, r2 ); \
1455+ r0 = _mm_unpacklo_epi16( r1, r3 ); \
1456+ r2 = _mm_unpackhi_epi16( r1, r3 ); \
1457+ r0 = _mm_packus_epi16( r0, r2 ); \
1458+ stbir__simdi_store( ptr, r0 ); \
1459+
1460+ #define stbir__simdi_32shr( out, reg, imm ) out = _mm_srli_epi32( reg, imm )
1461+
1462+ #if defined(_MSC_VER) && !defined(__clang__)
1463+ // msvc inits with 8 bytes
1464+ #define STBIR__CONST_32_TO_8( v ) (char)(unsigned char)((v)&255),(char)(unsigned char)(((v)>>8)&255),(char)(unsigned char)(((v)>>16)&255),(char)(unsigned char)(((v)>>24)&255)
1465+ #define STBIR__CONST_4_32i( v ) STBIR__CONST_32_TO_8( v ), STBIR__CONST_32_TO_8( v ), STBIR__CONST_32_TO_8( v ), STBIR__CONST_32_TO_8( v )
1466+ #define STBIR__CONST_4d_32i( v0, v1, v2, v3 ) STBIR__CONST_32_TO_8( v0 ), STBIR__CONST_32_TO_8( v1 ), STBIR__CONST_32_TO_8( v2 ), STBIR__CONST_32_TO_8( v3 )
1467+ #else
1468+ // everything else inits with long long's
1469+ #define STBIR__CONST_4_32i( v ) (long long)((((stbir_uint64)(stbir_uint32)(v))<<32)|((stbir_uint64)(stbir_uint32)(v))),(long long)((((stbir_uint64)(stbir_uint32)(v))<<32)|((stbir_uint64)(stbir_uint32)(v)))
1470+ #define STBIR__CONST_4d_32i( v0, v1, v2, v3 ) (long long)((((stbir_uint64)(stbir_uint32)(v1))<<32)|((stbir_uint64)(stbir_uint32)(v0))),(long long)((((stbir_uint64)(stbir_uint32)(v3))<<32)|((stbir_uint64)(stbir_uint32)(v2)))
1471+ #endif
1472+
1473+ #define STBIR__SIMDF_CONST(var, x) stbir__simdf var = { x, x, x, x }
1474+ #define STBIR__SIMDI_CONST(var, x) stbir__simdi var = { STBIR__CONST_4_32i(x) }
1475+ #define STBIR__CONSTF(var) (var)
1476+ #define STBIR__CONSTI(var) (var)
1477+
1478+ #if defined(STBIR_AVX) || defined(__SSE4_1__)
1479+ #include <smmintrin.h>
1480+ #define stbir__simdf_pack_to_8words(out,reg0,reg1) out = _mm_packus_epi32(_mm_cvttps_epi32(_mm_max_ps(_mm_min_ps(reg0,STBIR__CONSTF(STBIR_max_uint16_as_float)),_mm_setzero_ps())), _mm_cvttps_epi32(_mm_max_ps(_mm_min_ps(reg1,STBIR__CONSTF(STBIR_max_uint16_as_float)),_mm_setzero_ps())))
1481+ #else
1482+ static STBIR__SIMDI_CONST(stbir__s32_32768, 32768);
1483+ static STBIR__SIMDI_CONST(stbir__s16_32768, ((32768<<16)|32768));
1484+
1485+ #define stbir__simdf_pack_to_8words(out,reg0,reg1) \
1486+ { \
1487+ stbir__simdi tmp0,tmp1; \
1488+ tmp0 = _mm_cvttps_epi32(_mm_max_ps(_mm_min_ps(reg0,STBIR__CONSTF(STBIR_max_uint16_as_float)),_mm_setzero_ps())); \
1489+ tmp1 = _mm_cvttps_epi32(_mm_max_ps(_mm_min_ps(reg1,STBIR__CONSTF(STBIR_max_uint16_as_float)),_mm_setzero_ps())); \
1490+ tmp0 = _mm_sub_epi32( tmp0, stbir__s32_32768 ); \
1491+ tmp1 = _mm_sub_epi32( tmp1, stbir__s32_32768 ); \
1492+ out = _mm_packs_epi32( tmp0, tmp1 ); \
1493+ out = _mm_sub_epi16( out, stbir__s16_32768 ); \
1494+ }
1495+
1496+ #endif
1497+
1498+ #define STBIR_SIMD
1499+
1500+ // if we detect AVX, set the simd8 defines
1501+ #ifdef STBIR_AVX
1502+ #include <immintrin.h>
1503+ #define STBIR_SIMD8
1504+ #define stbir__simdf8 __m256
1505+ #define stbir__simdi8 __m256i
1506+ #define stbir__simdf8_load( out, ptr ) (out) = _mm256_loadu_ps( (float const *)(ptr) )
1507+ #define stbir__simdi8_load( out, ptr ) (out) = _mm256_loadu_si256( (__m256i const *)(ptr) )
1508+ #define stbir__simdf8_mult( out, a, b ) (out) = _mm256_mul_ps( (a), (b) )
1509+ #define stbir__simdf8_store( ptr, out ) _mm256_storeu_ps( (float*)(ptr), out )
1510+ #define stbir__simdi8_store( ptr, reg ) _mm256_storeu_si256( (__m256i*)(ptr), reg )
1511+ #define stbir__simdf8_frep8( fval ) _mm256_set1_ps( fval )
1512+
1513+ #define stbir__simdf8_min( out, reg0, reg1 ) (out) = _mm256_min_ps( reg0, reg1 )
1514+ #define stbir__simdf8_max( out, reg0, reg1 ) (out) = _mm256_max_ps( reg0, reg1 )
1515+
1516+ #define stbir__simdf8_add4halves( out, bot4, top8 ) (out) = _mm_add_ps( bot4, _mm256_extractf128_ps( top8, 1 ) )
1517+ #define stbir__simdf8_mult_mem( out, reg, ptr ) (out) = _mm256_mul_ps( reg, _mm256_loadu_ps( (float const*)(ptr) ) )
1518+ #define stbir__simdf8_add_mem( out, reg, ptr ) (out) = _mm256_add_ps( reg, _mm256_loadu_ps( (float const*)(ptr) ) )
1519+ #define stbir__simdf8_add( out, a, b ) (out) = _mm256_add_ps( a, b )
1520+ #define stbir__simdf8_load1b( out, ptr ) (out) = _mm256_broadcast_ss( ptr )
1521+ #define stbir__simdf_load1rep4( out, ptr ) (out) = _mm_broadcast_ss( ptr ) // avx load instruction
1522+
1523+ #define stbir__simdi8_convert_i32_to_float(out, ireg) (out) = _mm256_cvtepi32_ps( ireg )
1524+ #define stbir__simdf8_convert_float_to_i32( i, f ) (i) = _mm256_cvttps_epi32(f)
1525+
1526+ #define stbir__simdf8_bot4s( out, a, b ) (out) = _mm256_permute2f128_ps(a,b, (0<<0)+(2<<4) )
1527+ #define stbir__simdf8_top4s( out, a, b ) (out) = _mm256_permute2f128_ps(a,b, (1<<0)+(3<<4) )
1528+
1529+ #define stbir__simdf8_gettop4( reg ) _mm256_extractf128_ps(reg,1)
1530+
1531+ #ifdef STBIR_AVX2
1532+
1533+ #define stbir__simdi8_expand_u8_to_u32(out0,out1,ireg) \
1534+ { \
1535+ stbir__simdi8 a, zero =_mm256_setzero_si256();\
1536+ a = _mm256_permute4x64_epi64( _mm256_unpacklo_epi8( _mm256_permute4x64_epi64(_mm256_castsi128_si256(ireg),(0<<0)+(2<<2)+(1<<4)+(3<<6)), zero ),(0<<0)+(2<<2)+(1<<4)+(3<<6)); \
1537+ out0 = _mm256_unpacklo_epi16( a, zero ); \
1538+ out1 = _mm256_unpackhi_epi16( a, zero ); \
1539+ }
1540+
1541+ #define stbir__simdf8_pack_to_16bytes(out,aa,bb) \
1542+ { \
1543+ stbir__simdi8 t; \
1544+ stbir__simdf8 af,bf; \
1545+ stbir__simdi8 a,b; \
1546+ af = _mm256_min_ps( aa, STBIR_max_uint8_as_floatX ); \
1547+ bf = _mm256_min_ps( bb, STBIR_max_uint8_as_floatX ); \
1548+ af = _mm256_max_ps( af, _mm256_setzero_ps() ); \
1549+ bf = _mm256_max_ps( bf, _mm256_setzero_ps() ); \
1550+ a = _mm256_cvttps_epi32( af ); \
1551+ b = _mm256_cvttps_epi32( bf ); \
1552+ t = _mm256_permute4x64_epi64( _mm256_packs_epi32( a, b ), (0<<0)+(2<<2)+(1<<4)+(3<<6) ); \
1553+ out = _mm256_castsi256_si128( _mm256_permute4x64_epi64( _mm256_packus_epi16( t, t ), (0<<0)+(2<<2)+(1<<4)+(3<<6) ) ); \
1554+ }
1555+
1556+ #define stbir__simdi8_expand_u16_to_u32(out,ireg) out = _mm256_unpacklo_epi16( _mm256_permute4x64_epi64(_mm256_castsi128_si256(ireg),(0<<0)+(2<<2)+(1<<4)+(3<<6)), _mm256_setzero_si256() );
1557+
1558+ #define stbir__simdf8_pack_to_16words(out,aa,bb) \
1559+ { \
1560+ stbir__simdf8 af,bf; \
1561+ stbir__simdi8 a,b; \
1562+ af = _mm256_min_ps( aa, STBIR_max_uint16_as_floatX ); \
1563+ bf = _mm256_min_ps( bb, STBIR_max_uint16_as_floatX ); \
1564+ af = _mm256_max_ps( af, _mm256_setzero_ps() ); \
1565+ bf = _mm256_max_ps( bf, _mm256_setzero_ps() ); \
1566+ a = _mm256_cvttps_epi32( af ); \
1567+ b = _mm256_cvttps_epi32( bf ); \
1568+ (out) = _mm256_permute4x64_epi64( _mm256_packus_epi32(a, b), (0<<0)+(2<<2)+(1<<4)+(3<<6) ); \
1569+ }
1570+
1571+ #else
1572+
1573+ #define stbir__simdi8_expand_u8_to_u32(out0,out1,ireg) \
1574+ { \
1575+ stbir__simdi a,zero = _mm_setzero_si128(); \
1576+ a = _mm_unpacklo_epi8( ireg, zero ); \
1577+ out0 = _mm256_setr_m128i( _mm_unpacklo_epi16( a, zero ), _mm_unpackhi_epi16( a, zero ) ); \
1578+ a = _mm_unpackhi_epi8( ireg, zero ); \
1579+ out1 = _mm256_setr_m128i( _mm_unpacklo_epi16( a, zero ), _mm_unpackhi_epi16( a, zero ) ); \
1580+ }
1581+
1582+ #define stbir__simdf8_pack_to_16bytes(out,aa,bb) \
1583+ { \
1584+ stbir__simdi t; \
1585+ stbir__simdf8 af,bf; \
1586+ stbir__simdi8 a,b; \
1587+ af = _mm256_min_ps( aa, STBIR_max_uint8_as_floatX ); \
1588+ bf = _mm256_min_ps( bb, STBIR_max_uint8_as_floatX ); \
1589+ af = _mm256_max_ps( af, _mm256_setzero_ps() ); \
1590+ bf = _mm256_max_ps( bf, _mm256_setzero_ps() ); \
1591+ a = _mm256_cvttps_epi32( af ); \
1592+ b = _mm256_cvttps_epi32( bf ); \
1593+ out = _mm_packs_epi32( _mm256_castsi256_si128(a), _mm256_extractf128_si256( a, 1 ) ); \
1594+ out = _mm_packus_epi16( out, out ); \
1595+ t = _mm_packs_epi32( _mm256_castsi256_si128(b), _mm256_extractf128_si256( b, 1 ) ); \
1596+ t = _mm_packus_epi16( t, t ); \
1597+ out = _mm_castps_si128( _mm_shuffle_ps( _mm_castsi128_ps(out), _mm_castsi128_ps(t), (0<<0)+(1<<2)+(0<<4)+(1<<6) ) ); \
1598+ }
1599+
1600+ #define stbir__simdi8_expand_u16_to_u32(out,ireg) \
1601+ { \
1602+ stbir__simdi a,b,zero = _mm_setzero_si128(); \
1603+ a = _mm_unpacklo_epi16( ireg, zero ); \
1604+ b = _mm_unpackhi_epi16( ireg, zero ); \
1605+ out = _mm256_insertf128_si256( _mm256_castsi128_si256( a ), b, 1 ); \
1606+ }
1607+
1608+ #define stbir__simdf8_pack_to_16words(out,aa,bb) \
1609+ { \
1610+ stbir__simdi t0,t1; \
1611+ stbir__simdf8 af,bf; \
1612+ stbir__simdi8 a,b; \
1613+ af = _mm256_min_ps( aa, STBIR_max_uint16_as_floatX ); \
1614+ bf = _mm256_min_ps( bb, STBIR_max_uint16_as_floatX ); \
1615+ af = _mm256_max_ps( af, _mm256_setzero_ps() ); \
1616+ bf = _mm256_max_ps( bf, _mm256_setzero_ps() ); \
1617+ a = _mm256_cvttps_epi32( af ); \
1618+ b = _mm256_cvttps_epi32( bf ); \
1619+ t0 = _mm_packus_epi32( _mm256_castsi256_si128(a), _mm256_extractf128_si256( a, 1 ) ); \
1620+ t1 = _mm_packus_epi32( _mm256_castsi256_si128(b), _mm256_extractf128_si256( b, 1 ) ); \
1621+ out = _mm256_setr_m128i( t0, t1 ); \
1622+ }
1623+
1624+ #endif
1625+
1626+ static __m256i stbir_00001111 = { STBIR__CONST_4d_32i( 0, 0, 0, 0 ), STBIR__CONST_4d_32i( 1, 1, 1, 1 ) };
1627+ #define stbir__simdf8_0123to00001111( out, in ) (out) = _mm256_permutevar_ps ( in, stbir_00001111 )
1628+
1629+ static __m256i stbir_22223333 = { STBIR__CONST_4d_32i( 2, 2, 2, 2 ), STBIR__CONST_4d_32i( 3, 3, 3, 3 ) };
1630+ #define stbir__simdf8_0123to22223333( out, in ) (out) = _mm256_permutevar_ps ( in, stbir_22223333 )
1631+
1632+ #define stbir__simdf8_0123to2222( out, in ) (out) = stbir__simdf_swiz(_mm256_castps256_ps128(in), 2,2,2,2 )
1633+
1634+ #define stbir__simdf8_load4b( out, ptr ) (out) = _mm256_broadcast_ps( (__m128 const *)(ptr) )
1635+
1636+ static __m256i stbir_00112233 = { STBIR__CONST_4d_32i( 0, 0, 1, 1 ), STBIR__CONST_4d_32i( 2, 2, 3, 3 ) };
1637+ #define stbir__simdf8_0123to00112233( out, in ) (out) = _mm256_permutevar_ps ( in, stbir_00112233 )
1638+ #define stbir__simdf8_add4( out, a8, b ) (out) = _mm256_add_ps( a8, _mm256_castps128_ps256( b ) )
1639+
1640+ static __m256i stbir_load6 = { STBIR__CONST_4_32i( 0x80000000 ), STBIR__CONST_4d_32i( 0x80000000, 0x80000000, 0, 0 ) };
1641+ #define stbir__simdf8_load6z( out, ptr ) (out) = _mm256_maskload_ps( ptr, stbir_load6 )
1642+
1643+ #define stbir__simdf8_0123to00000000( out, in ) (out) = _mm256_shuffle_ps ( in, in, (0<<0)+(0<<2)+(0<<4)+(0<<6) )
1644+ #define stbir__simdf8_0123to11111111( out, in ) (out) = _mm256_shuffle_ps ( in, in, (1<<0)+(1<<2)+(1<<4)+(1<<6) )
1645+ #define stbir__simdf8_0123to22222222( out, in ) (out) = _mm256_shuffle_ps ( in, in, (2<<0)+(2<<2)+(2<<4)+(2<<6) )
1646+ #define stbir__simdf8_0123to33333333( out, in ) (out) = _mm256_shuffle_ps ( in, in, (3<<0)+(3<<2)+(3<<4)+(3<<6) )
1647+ #define stbir__simdf8_0123to21032103( out, in ) (out) = _mm256_shuffle_ps ( in, in, (2<<0)+(1<<2)+(0<<4)+(3<<6) )
1648+ #define stbir__simdf8_0123to32103210( out, in ) (out) = _mm256_shuffle_ps ( in, in, (3<<0)+(2<<2)+(1<<4)+(0<<6) )
1649+ #define stbir__simdf8_0123to12301230( out, in ) (out) = _mm256_shuffle_ps ( in, in, (1<<0)+(2<<2)+(3<<4)+(0<<6) )
1650+ #define stbir__simdf8_0123to10321032( out, in ) (out) = _mm256_shuffle_ps ( in, in, (1<<0)+(0<<2)+(3<<4)+(2<<6) )
1651+ #define stbir__simdf8_0123to30123012( out, in ) (out) = _mm256_shuffle_ps ( in, in, (3<<0)+(0<<2)+(1<<4)+(2<<6) )
1652+
1653+ #define stbir__simdf8_0123to11331133( out, in ) (out) = _mm256_shuffle_ps ( in, in, (1<<0)+(1<<2)+(3<<4)+(3<<6) )
1654+ #define stbir__simdf8_0123to00220022( out, in ) (out) = _mm256_shuffle_ps ( in, in, (0<<0)+(0<<2)+(2<<4)+(2<<6) )
1655+
1656+ #define stbir__simdf8_aaa1( out, alp, ones ) (out) = _mm256_blend_ps( alp, ones, (1<<0)+(1<<1)+(1<<2)+(0<<3)+(1<<4)+(1<<5)+(1<<6)+(0<<7)); (out)=_mm256_shuffle_ps( out,out, (3<<0) + (3<<2) + (3<<4) + (0<<6) )
1657+ #define stbir__simdf8_1aaa( out, alp, ones ) (out) = _mm256_blend_ps( alp, ones, (0<<0)+(1<<1)+(1<<2)+(1<<3)+(0<<4)+(1<<5)+(1<<6)+(1<<7)); (out)=_mm256_shuffle_ps( out,out, (1<<0) + (0<<2) + (0<<4) + (0<<6) )
1658+ #define stbir__simdf8_a1a1( out, alp, ones) (out) = _mm256_blend_ps( alp, ones, (1<<0)+(0<<1)+(1<<2)+(0<<3)+(1<<4)+(0<<5)+(1<<6)+(0<<7)); (out)=_mm256_shuffle_ps( out,out, (1<<0) + (0<<2) + (3<<4) + (2<<6) )
1659+ #define stbir__simdf8_1a1a( out, alp, ones) (out) = _mm256_blend_ps( alp, ones, (0<<0)+(1<<1)+(0<<2)+(1<<3)+(0<<4)+(1<<5)+(0<<6)+(1<<7)); (out)=_mm256_shuffle_ps( out,out, (1<<0) + (0<<2) + (3<<4) + (2<<6) )
1660+
1661+ #define stbir__simdf8_zero( reg ) (reg) = _mm256_setzero_ps()
1662+
1663+ #ifdef STBIR_USE_FMA // not on by default to maintain bit identical simd to non-simd
1664+ #define stbir__simdf8_madd( out, add, mul1, mul2 ) (out) = _mm256_fmadd_ps( mul1, mul2, add )
1665+ #define stbir__simdf8_madd_mem( out, add, mul, ptr ) (out) = _mm256_fmadd_ps( mul, _mm256_loadu_ps( (float const*)(ptr) ), add )
1666+ #define stbir__simdf8_madd_mem4( out, add, mul, ptr )(out) = _mm256_fmadd_ps( _mm256_setr_m128( mul, _mm_setzero_ps() ), _mm256_setr_m128( _mm_loadu_ps( (float const*)(ptr) ), _mm_setzero_ps() ), add )
1667+ #else
1668+ #define stbir__simdf8_madd( out, add, mul1, mul2 ) (out) = _mm256_add_ps( add, _mm256_mul_ps( mul1, mul2 ) )
1669+ #define stbir__simdf8_madd_mem( out, add, mul, ptr ) (out) = _mm256_add_ps( add, _mm256_mul_ps( mul, _mm256_loadu_ps( (float const*)(ptr) ) ) )
1670+ #define stbir__simdf8_madd_mem4( out, add, mul, ptr ) (out) = _mm256_add_ps( add, _mm256_setr_m128( _mm_mul_ps( mul, _mm_loadu_ps( (float const*)(ptr) ) ), _mm_setzero_ps() ) )
1671+ #endif
1672+ #define stbir__if_simdf8_cast_to_simdf4( val ) _mm256_castps256_ps128( val )
1673+
1674+ #endif
1675+
1676+ #ifdef STBIR_FLOORF
1677+ #undef STBIR_FLOORF
1678+ #endif
1679+ #define STBIR_FLOORF stbir_simd_floorf
1680+ static stbir__inline float stbir_simd_floorf(float x) // martins floorf
1681+ {
1682+ #if defined(STBIR_AVX) || defined(__SSE4_1__) || defined(STBIR_SSE41)
1683+ __m128 t = _mm_set_ss(x);
1684+ return _mm_cvtss_f32( _mm_floor_ss(t, t) );
1685+ #else
1686+ __m128 f = _mm_set_ss(x);
1687+ __m128 t = _mm_cvtepi32_ps(_mm_cvttps_epi32(f));
1688+ __m128 r = _mm_add_ss(t, _mm_and_ps(_mm_cmplt_ss(f, t), _mm_set_ss(-1.0f)));
1689+ return _mm_cvtss_f32(r);
1690+ #endif
1691+ }
1692+
1693+ #ifdef STBIR_CEILF
1694+ #undef STBIR_CEILF
1695+ #endif
1696+ #define STBIR_CEILF stbir_simd_ceilf
1697+ static stbir__inline float stbir_simd_ceilf(float x) // martins ceilf
1698+ {
1699+ #if defined(STBIR_AVX) || defined(__SSE4_1__) || defined(STBIR_SSE41)
1700+ __m128 t = _mm_set_ss(x);
1701+ return _mm_cvtss_f32( _mm_ceil_ss(t, t) );
1702+ #else
1703+ __m128 f = _mm_set_ss(x);
1704+ __m128 t = _mm_cvtepi32_ps(_mm_cvttps_epi32(f));
1705+ __m128 r = _mm_add_ss(t, _mm_and_ps(_mm_cmplt_ss(t, f), _mm_set_ss(1.0f)));
1706+ return _mm_cvtss_f32(r);
1707+ #endif
1708+ }
1709+
1710+#elif defined(STBIR_NEON)
1711+
1712+ #include <arm_neon.h>
1713+
1714+ #define stbir__simdf float32x4_t
1715+ #define stbir__simdi uint32x4_t
1716+
1717+ #define stbir_simdi_castf( reg ) vreinterpretq_u32_f32(reg)
1718+ #define stbir_simdf_casti( reg ) vreinterpretq_f32_u32(reg)
1719+
1720+ #define stbir__simdf_load( reg, ptr ) (reg) = vld1q_f32( (float const*)(ptr) )
1721+ #define stbir__simdi_load( reg, ptr ) (reg) = vld1q_u32( (uint32_t const*)(ptr) )
1722+ #define stbir__simdf_load1( out, ptr ) (out) = vld1q_dup_f32( (float const*)(ptr) ) // top values can be random (not denormal or nan for perf)
1723+ #define stbir__simdi_load1( out, ptr ) (out) = vld1q_dup_u32( (uint32_t const*)(ptr) )
1724+ #define stbir__simdf_load1z( out, ptr ) (out) = vld1q_lane_f32( (float const*)(ptr), vdupq_n_f32(0), 0 ) // top values must be zero
1725+ #define stbir__simdf_frep4( fvar ) vdupq_n_f32( fvar )
1726+ #define stbir__simdf_load1frep4( out, fvar ) (out) = vdupq_n_f32( fvar )
1727+ #define stbir__simdf_load2( out, ptr ) (out) = vcombine_f32( vld1_f32( (float const*)(ptr) ), vcreate_f32(0) ) // top values can be random (not denormal or nan for perf)
1728+ #define stbir__simdf_load2z( out, ptr ) (out) = vcombine_f32( vld1_f32( (float const*)(ptr) ), vcreate_f32(0) ) // top values must be zero
1729+ #define stbir__simdf_load2hmerge( out, reg, ptr ) (out) = vcombine_f32( vget_low_f32(reg), vld1_f32( (float const*)(ptr) ) )
1730+
1731+ #define stbir__simdf_zeroP() vdupq_n_f32(0)
1732+ #define stbir__simdf_zero( reg ) (reg) = vdupq_n_f32(0)
1733+
1734+ #define stbir__simdf_store( ptr, reg ) vst1q_f32( (float*)(ptr), reg )
1735+ #define stbir__simdf_store1( ptr, reg ) vst1q_lane_f32( (float*)(ptr), reg, 0)
1736+ #define stbir__simdf_store2( ptr, reg ) vst1_f32( (float*)(ptr), vget_low_f32(reg) )
1737+ #define stbir__simdf_store2h( ptr, reg ) vst1_f32( (float*)(ptr), vget_high_f32(reg) )
1738+
1739+ #define stbir__simdi_store( ptr, reg ) vst1q_u32( (uint32_t*)(ptr), reg )
1740+ #define stbir__simdi_store1( ptr, reg ) vst1q_lane_u32( (uint32_t*)(ptr), reg, 0 )
1741+ #define stbir__simdi_store2( ptr, reg ) vst1_u32( (uint32_t*)(ptr), vget_low_u32(reg) )
1742+
1743+ #define stbir__prefetch( ptr )
1744+
1745+ #define stbir__simdi_expand_u8_to_u32(out0,out1,out2,out3,ireg) \
1746+ { \
1747+ uint16x8_t l = vmovl_u8( vget_low_u8 ( vreinterpretq_u8_u32(ireg) ) ); \
1748+ uint16x8_t h = vmovl_u8( vget_high_u8( vreinterpretq_u8_u32(ireg) ) ); \
1749+ out0 = vmovl_u16( vget_low_u16 ( l ) ); \
1750+ out1 = vmovl_u16( vget_high_u16( l ) ); \
1751+ out2 = vmovl_u16( vget_low_u16 ( h ) ); \
1752+ out3 = vmovl_u16( vget_high_u16( h ) ); \
1753+ }
1754+
1755+ #define stbir__simdi_expand_u8_to_1u32(out,ireg) \
1756+ { \
1757+ uint16x8_t tmp = vmovl_u8( vget_low_u8( vreinterpretq_u8_u32(ireg) ) ); \
1758+ out = vmovl_u16( vget_low_u16( tmp ) ); \
1759+ }
1760+
1761+ #define stbir__simdi_expand_u16_to_u32(out0,out1,ireg) \
1762+ { \
1763+ uint16x8_t tmp = vreinterpretq_u16_u32(ireg); \
1764+ out0 = vmovl_u16( vget_low_u16 ( tmp ) ); \
1765+ out1 = vmovl_u16( vget_high_u16( tmp ) ); \
1766+ }
1767+
1768+ #define stbir__simdf_convert_float_to_i32( i, f ) (i) = vreinterpretq_u32_s32( vcvtq_s32_f32(f) )
1769+ #define stbir__simdf_convert_float_to_int( f ) vgetq_lane_s32(vcvtq_s32_f32(f), 0)
1770+ #define stbir__simdi_to_int( i ) (int)vgetq_lane_u32(i, 0)
1771+ #define stbir__simdf_convert_float_to_uint8( f ) ((unsigned char)vgetq_lane_s32(vcvtq_s32_f32(vmaxq_f32(vminq_f32(f,STBIR__CONSTF(STBIR_max_uint8_as_float)),vdupq_n_f32(0))), 0))
1772+ #define stbir__simdf_convert_float_to_short( f ) ((unsigned short)vgetq_lane_s32(vcvtq_s32_f32(vmaxq_f32(vminq_f32(f,STBIR__CONSTF(STBIR_max_uint16_as_float)),vdupq_n_f32(0))), 0))
1773+ #define stbir__simdi_convert_i32_to_float(out, ireg) (out) = vcvtq_f32_s32( vreinterpretq_s32_u32(ireg) )
1774+ #define stbir__simdf_add( out, reg0, reg1 ) (out) = vaddq_f32( reg0, reg1 )
1775+ #define stbir__simdf_mult( out, reg0, reg1 ) (out) = vmulq_f32( reg0, reg1 )
1776+ #define stbir__simdf_mult_mem( out, reg, ptr ) (out) = vmulq_f32( reg, vld1q_f32( (float const*)(ptr) ) )
1777+ #define stbir__simdf_mult1_mem( out, reg, ptr ) (out) = vmulq_f32( reg, vld1q_dup_f32( (float const*)(ptr) ) )
1778+ #define stbir__simdf_add_mem( out, reg, ptr ) (out) = vaddq_f32( reg, vld1q_f32( (float const*)(ptr) ) )
1779+ #define stbir__simdf_add1_mem( out, reg, ptr ) (out) = vaddq_f32( reg, vld1q_dup_f32( (float const*)(ptr) ) )
1780+
1781+ #ifdef STBIR_USE_FMA // not on by default to maintain bit identical simd to non-simd (and also x64 no madd to arm madd)
1782+ #define stbir__simdf_madd( out, add, mul1, mul2 ) (out) = vfmaq_f32( add, mul1, mul2 )
1783+ #define stbir__simdf_madd1( out, add, mul1, mul2 ) (out) = vfmaq_f32( add, mul1, mul2 )
1784+ #define stbir__simdf_madd_mem( out, add, mul, ptr ) (out) = vfmaq_f32( add, mul, vld1q_f32( (float const*)(ptr) ) )
1785+ #define stbir__simdf_madd1_mem( out, add, mul, ptr ) (out) = vfmaq_f32( add, mul, vld1q_dup_f32( (float const*)(ptr) ) )
1786+ #else
1787+ #define stbir__simdf_madd( out, add, mul1, mul2 ) (out) = vaddq_f32( add, vmulq_f32( mul1, mul2 ) )
1788+ #define stbir__simdf_madd1( out, add, mul1, mul2 ) (out) = vaddq_f32( add, vmulq_f32( mul1, mul2 ) )
1789+ #define stbir__simdf_madd_mem( out, add, mul, ptr ) (out) = vaddq_f32( add, vmulq_f32( mul, vld1q_f32( (float const*)(ptr) ) ) )
1790+ #define stbir__simdf_madd1_mem( out, add, mul, ptr ) (out) = vaddq_f32( add, vmulq_f32( mul, vld1q_dup_f32( (float const*)(ptr) ) ) )
1791+ #endif
1792+
1793+ #define stbir__simdf_add1( out, reg0, reg1 ) (out) = vaddq_f32( reg0, reg1 )
1794+ #define stbir__simdf_mult1( out, reg0, reg1 ) (out) = vmulq_f32( reg0, reg1 )
1795+
1796+ #define stbir__simdf_and( out, reg0, reg1 ) (out) = vreinterpretq_f32_u32( vandq_u32( vreinterpretq_u32_f32(reg0), vreinterpretq_u32_f32(reg1) ) )
1797+ #define stbir__simdf_or( out, reg0, reg1 ) (out) = vreinterpretq_f32_u32( vorrq_u32( vreinterpretq_u32_f32(reg0), vreinterpretq_u32_f32(reg1) ) )
1798+
1799+ #define stbir__simdf_min( out, reg0, reg1 ) (out) = vminq_f32( reg0, reg1 )
1800+ #define stbir__simdf_max( out, reg0, reg1 ) (out) = vmaxq_f32( reg0, reg1 )
1801+ #define stbir__simdf_min1( out, reg0, reg1 ) (out) = vminq_f32( reg0, reg1 )
1802+ #define stbir__simdf_max1( out, reg0, reg1 ) (out) = vmaxq_f32( reg0, reg1 )
1803+
1804+ #define stbir__simdf_0123ABCDto3ABx( out, reg0, reg1 ) (out) = vextq_f32( reg0, reg1, 3 )
1805+ #define stbir__simdf_0123ABCDto23Ax( out, reg0, reg1 ) (out) = vextq_f32( reg0, reg1, 2 )
1806+
1807+ #define stbir__simdf_a1a1( out, alp, ones ) (out) = vzipq_f32(vuzpq_f32(alp, alp).val[1], ones).val[0]
1808+ #define stbir__simdf_1a1a( out, alp, ones ) (out) = vzipq_f32(ones, vuzpq_f32(alp, alp).val[0]).val[0]
1809+
1810+ #if defined( _M_ARM64 ) || defined( __aarch64__ ) || defined( __arm64__ )
1811+
1812+ #define stbir__simdf_aaa1( out, alp, ones ) (out) = vcopyq_laneq_f32(vdupq_n_f32(vgetq_lane_f32(alp, 3)), 3, ones, 3)
1813+ #define stbir__simdf_1aaa( out, alp, ones ) (out) = vcopyq_laneq_f32(vdupq_n_f32(vgetq_lane_f32(alp, 0)), 0, ones, 0)
1814+
1815+ #if defined( _MSC_VER ) && !defined(__clang__)
1816+ #define stbir_make16(a,b,c,d) vcombine_u8( \
1817+ vcreate_u8( (4*a+0) | ((4*a+1)<<8) | ((4*a+2)<<16) | ((4*a+3)<<24) | \
1818+ ((stbir_uint64)(4*b+0)<<32) | ((stbir_uint64)(4*b+1)<<40) | ((stbir_uint64)(4*b+2)<<48) | ((stbir_uint64)(4*b+3)<<56)), \
1819+ vcreate_u8( (4*c+0) | ((4*c+1)<<8) | ((4*c+2)<<16) | ((4*c+3)<<24) | \
1820+ ((stbir_uint64)(4*d+0)<<32) | ((stbir_uint64)(4*d+1)<<40) | ((stbir_uint64)(4*d+2)<<48) | ((stbir_uint64)(4*d+3)<<56) ) )
1821+
1822+ static stbir__inline uint8x16x2_t stbir_make16x2(float32x4_t rega,float32x4_t regb)
1823+ {
1824+ uint8x16x2_t r = { vreinterpretq_u8_f32(rega), vreinterpretq_u8_f32(regb) };
1825+ return r;
1826+ }
1827+ #else
1828+ #define stbir_make16(a,b,c,d) (uint8x16_t){4*a+0,4*a+1,4*a+2,4*a+3,4*b+0,4*b+1,4*b+2,4*b+3,4*c+0,4*c+1,4*c+2,4*c+3,4*d+0,4*d+1,4*d+2,4*d+3}
1829+ #define stbir_make16x2(a,b) (uint8x16x2_t){{vreinterpretq_u8_f32(a),vreinterpretq_u8_f32(b)}}
1830+ #endif
1831+
1832+ #define stbir__simdf_swiz( reg, one, two, three, four ) vreinterpretq_f32_u8( vqtbl1q_u8( vreinterpretq_u8_f32(reg), stbir_make16(one, two, three, four) ) )
1833+ #define stbir__simdf_swiz2( rega, regb, one, two, three, four ) vreinterpretq_f32_u8( vqtbl2q_u8( stbir_make16x2(rega,regb), stbir_make16(one, two, three, four) ) )
1834+
1835+ #define stbir__simdi_16madd( out, reg0, reg1 ) \
1836+ { \
1837+ int16x8_t r0 = vreinterpretq_s16_u32(reg0); \
1838+ int16x8_t r1 = vreinterpretq_s16_u32(reg1); \
1839+ int32x4_t tmp0 = vmull_s16( vget_low_s16(r0), vget_low_s16(r1) ); \
1840+ int32x4_t tmp1 = vmull_s16( vget_high_s16(r0), vget_high_s16(r1) ); \
1841+ (out) = vreinterpretq_u32_s32( vpaddq_s32(tmp0, tmp1) ); \
1842+ }
1843+
1844+ #else
1845+
1846+ #define stbir__simdf_aaa1( out, alp, ones ) (out) = vsetq_lane_f32(1.0f, vdupq_n_f32(vgetq_lane_f32(alp, 3)), 3)
1847+ #define stbir__simdf_1aaa( out, alp, ones ) (out) = vsetq_lane_f32(1.0f, vdupq_n_f32(vgetq_lane_f32(alp, 0)), 0)
1848+
1849+ #if defined( _MSC_VER ) && !defined(__clang__)
1850+ static stbir__inline uint8x8x2_t stbir_make8x2(float32x4_t reg)
1851+ {
1852+ uint8x8x2_t r = { { vget_low_u8(vreinterpretq_u8_f32(reg)), vget_high_u8(vreinterpretq_u8_f32(reg)) } };
1853+ return r;
1854+ }
1855+ #define stbir_make8(a,b) vcreate_u8( \
1856+ (4*a+0) | ((4*a+1)<<8) | ((4*a+2)<<16) | ((4*a+3)<<24) | \
1857+ ((stbir_uint64)(4*b+0)<<32) | ((stbir_uint64)(4*b+1)<<40) | ((stbir_uint64)(4*b+2)<<48) | ((stbir_uint64)(4*b+3)<<56) )
1858+ #else
1859+ #define stbir_make8x2(reg) (uint8x8x2_t){ { vget_low_u8(vreinterpretq_u8_f32(reg)), vget_high_u8(vreinterpretq_u8_f32(reg)) } }
1860+ #define stbir_make8(a,b) (uint8x8_t){4*a+0,4*a+1,4*a+2,4*a+3,4*b+0,4*b+1,4*b+2,4*b+3}
1861+ #endif
1862+
1863+ #define stbir__simdf_swiz( reg, one, two, three, four ) vreinterpretq_f32_u8( vcombine_u8( \
1864+ vtbl2_u8( stbir_make8x2( reg ), stbir_make8( one, two ) ), \
1865+ vtbl2_u8( stbir_make8x2( reg ), stbir_make8( three, four ) ) ) )
1866+
1867+ #define stbir__simdi_16madd( out, reg0, reg1 ) \
1868+ { \
1869+ int16x8_t r0 = vreinterpretq_s16_u32(reg0); \
1870+ int16x8_t r1 = vreinterpretq_s16_u32(reg1); \
1871+ int32x4_t tmp0 = vmull_s16( vget_low_s16(r0), vget_low_s16(r1) ); \
1872+ int32x4_t tmp1 = vmull_s16( vget_high_s16(r0), vget_high_s16(r1) ); \
1873+ int32x2_t out0 = vpadd_s32( vget_low_s32(tmp0), vget_high_s32(tmp0) ); \
1874+ int32x2_t out1 = vpadd_s32( vget_low_s32(tmp1), vget_high_s32(tmp1) ); \
1875+ (out) = vreinterpretq_u32_s32( vcombine_s32(out0, out1) ); \
1876+ }
1877+
1878+ #endif
1879+
1880+ #define stbir__simdi_and( out, reg0, reg1 ) (out) = vandq_u32( reg0, reg1 )
1881+ #define stbir__simdi_or( out, reg0, reg1 ) (out) = vorrq_u32( reg0, reg1 )
1882+
1883+ #define stbir__simdf_pack_to_8bytes(out,aa,bb) \
1884+ { \
1885+ float32x4_t af = vmaxq_f32( vminq_f32(aa,STBIR__CONSTF(STBIR_max_uint8_as_float) ), vdupq_n_f32(0) ); \
1886+ float32x4_t bf = vmaxq_f32( vminq_f32(bb,STBIR__CONSTF(STBIR_max_uint8_as_float) ), vdupq_n_f32(0) ); \
1887+ int16x4_t ai = vqmovn_s32( vcvtq_s32_f32( af ) ); \
1888+ int16x4_t bi = vqmovn_s32( vcvtq_s32_f32( bf ) ); \
1889+ uint8x8_t out8 = vqmovun_s16( vcombine_s16(ai, bi) ); \
1890+ out = vreinterpretq_u32_u8( vcombine_u8(out8, out8) ); \
1891+ }
1892+
1893+ #define stbir__simdf_pack_to_8words(out,aa,bb) \
1894+ { \
1895+ float32x4_t af = vmaxq_f32( vminq_f32(aa,STBIR__CONSTF(STBIR_max_uint16_as_float) ), vdupq_n_f32(0) ); \
1896+ float32x4_t bf = vmaxq_f32( vminq_f32(bb,STBIR__CONSTF(STBIR_max_uint16_as_float) ), vdupq_n_f32(0) ); \
1897+ int32x4_t ai = vcvtq_s32_f32( af ); \
1898+ int32x4_t bi = vcvtq_s32_f32( bf ); \
1899+ out = vreinterpretq_u32_u16( vcombine_u16(vqmovun_s32(ai), vqmovun_s32(bi)) ); \
1900+ }
1901+
1902+ #define stbir__interleave_pack_and_store_16_u8( ptr, r0, r1, r2, r3 ) \
1903+ { \
1904+ int16x4x2_t tmp0 = vzip_s16( vqmovn_s32(vreinterpretq_s32_u32(r0)), vqmovn_s32(vreinterpretq_s32_u32(r2)) ); \
1905+ int16x4x2_t tmp1 = vzip_s16( vqmovn_s32(vreinterpretq_s32_u32(r1)), vqmovn_s32(vreinterpretq_s32_u32(r3)) ); \
1906+ uint8x8x2_t out = \
1907+ { { \
1908+ vqmovun_s16( vcombine_s16(tmp0.val[0], tmp0.val[1]) ), \
1909+ vqmovun_s16( vcombine_s16(tmp1.val[0], tmp1.val[1]) ), \
1910+ } }; \
1911+ vst2_u8(ptr, out); \
1912+ }
1913+
1914+ #define stbir__simdf_load4_transposed( o0, o1, o2, o3, ptr ) \
1915+ { \
1916+ float32x4x4_t tmp = vld4q_f32(ptr); \
1917+ o0 = tmp.val[0]; \
1918+ o1 = tmp.val[1]; \
1919+ o2 = tmp.val[2]; \
1920+ o3 = tmp.val[3]; \
1921+ }
1922+
1923+ #define stbir__simdi_32shr( out, reg, imm ) out = vshrq_n_u32( reg, imm )
1924+
1925+ #if defined( _MSC_VER ) && !defined(__clang__)
1926+ #define STBIR__SIMDF_CONST(var, x) __declspec(align(8)) float var[] = { x, x, x, x }
1927+ #define STBIR__SIMDI_CONST(var, x) __declspec(align(8)) uint32_t var[] = { x, x, x, x }
1928+ #define STBIR__CONSTF(var) (*(const float32x4_t*)var)
1929+ #define STBIR__CONSTI(var) (*(const uint32x4_t*)var)
1930+ #else
1931+ #define STBIR__SIMDF_CONST(var, x) stbir__simdf var = { x, x, x, x }
1932+ #define STBIR__SIMDI_CONST(var, x) stbir__simdi var = { x, x, x, x }
1933+ #define STBIR__CONSTF(var) (var)
1934+ #define STBIR__CONSTI(var) (var)
1935+ #endif
1936+
1937+ #ifdef STBIR_FLOORF
1938+ #undef STBIR_FLOORF
1939+ #endif
1940+ #define STBIR_FLOORF stbir_simd_floorf
1941+ static stbir__inline float stbir_simd_floorf(float x)
1942+ {
1943+ #if defined( _M_ARM64 ) || defined( __aarch64__ ) || defined( __arm64__ )
1944+ return vget_lane_f32( vrndm_f32( vdup_n_f32(x) ), 0);
1945+ #else
1946+ float32x2_t f = vdup_n_f32(x);
1947+ float32x2_t t = vcvt_f32_s32(vcvt_s32_f32(f));
1948+ uint32x2_t a = vclt_f32(f, t);
1949+ uint32x2_t b = vreinterpret_u32_f32(vdup_n_f32(-1.0f));
1950+ float32x2_t r = vadd_f32(t, vreinterpret_f32_u32(vand_u32(a, b)));
1951+ return vget_lane_f32(r, 0);
1952+ #endif
1953+ }
1954+
1955+ #ifdef STBIR_CEILF
1956+ #undef STBIR_CEILF
1957+ #endif
1958+ #define STBIR_CEILF stbir_simd_ceilf
1959+ static stbir__inline float stbir_simd_ceilf(float x)
1960+ {
1961+ #if defined( _M_ARM64 ) || defined( __aarch64__ ) || defined( __arm64__ )
1962+ return vget_lane_f32( vrndp_f32( vdup_n_f32(x) ), 0);
1963+ #else
1964+ float32x2_t f = vdup_n_f32(x);
1965+ float32x2_t t = vcvt_f32_s32(vcvt_s32_f32(f));
1966+ uint32x2_t a = vclt_f32(t, f);
1967+ uint32x2_t b = vreinterpret_u32_f32(vdup_n_f32(1.0f));
1968+ float32x2_t r = vadd_f32(t, vreinterpret_f32_u32(vand_u32(a, b)));
1969+ return vget_lane_f32(r, 0);
1970+ #endif
1971+ }
1972+
1973+ #define STBIR_SIMD
1974+
1975+#elif defined(STBIR_WASM)
1976+
1977+ #include <wasm_simd128.h>
1978+
1979+ #define stbir__simdf v128_t
1980+ #define stbir__simdi v128_t
1981+
1982+ #define stbir_simdi_castf( reg ) (reg)
1983+ #define stbir_simdf_casti( reg ) (reg)
1984+
1985+ #define stbir__simdf_load( reg, ptr ) (reg) = wasm_v128_load( (void const*)(ptr) )
1986+ #define stbir__simdi_load( reg, ptr ) (reg) = wasm_v128_load( (void const*)(ptr) )
1987+ #define stbir__simdf_load1( out, ptr ) (out) = wasm_v128_load32_splat( (void const*)(ptr) ) // top values can be random (not denormal or nan for perf)
1988+ #define stbir__simdi_load1( out, ptr ) (out) = wasm_v128_load32_splat( (void const*)(ptr) )
1989+ #define stbir__simdf_load1z( out, ptr ) (out) = wasm_v128_load32_zero( (void const*)(ptr) ) // top values must be zero
1990+ #define stbir__simdf_frep4( fvar ) wasm_f32x4_splat( fvar )
1991+ #define stbir__simdf_load1frep4( out, fvar ) (out) = wasm_f32x4_splat( fvar )
1992+ #define stbir__simdf_load2( out, ptr ) (out) = wasm_v128_load64_splat( (void const*)(ptr) ) // top values can be random (not denormal or nan for perf)
1993+ #define stbir__simdf_load2z( out, ptr ) (out) = wasm_v128_load64_zero( (void const*)(ptr) ) // top values must be zero
1994+ #define stbir__simdf_load2hmerge( out, reg, ptr ) (out) = wasm_v128_load64_lane( (void const*)(ptr), reg, 1 )
1995+
1996+ #define stbir__simdf_zeroP() wasm_f32x4_const_splat(0)
1997+ #define stbir__simdf_zero( reg ) (reg) = wasm_f32x4_const_splat(0)
1998+
1999+ #define stbir__simdf_store( ptr, reg ) wasm_v128_store( (void*)(ptr), reg )
2000+ #define stbir__simdf_store1( ptr, reg ) wasm_v128_store32_lane( (void*)(ptr), reg, 0 )
2001+ #define stbir__simdf_store2( ptr, reg ) wasm_v128_store64_lane( (void*)(ptr), reg, 0 )
2002+ #define stbir__simdf_store2h( ptr, reg ) wasm_v128_store64_lane( (void*)(ptr), reg, 1 )
2003+
2004+ #define stbir__simdi_store( ptr, reg ) wasm_v128_store( (void*)(ptr), reg )
2005+ #define stbir__simdi_store1( ptr, reg ) wasm_v128_store32_lane( (void*)(ptr), reg, 0 )
2006+ #define stbir__simdi_store2( ptr, reg ) wasm_v128_store64_lane( (void*)(ptr), reg, 0 )
2007+
2008+ #define stbir__prefetch( ptr )
2009+
2010+ #define stbir__simdi_expand_u8_to_u32(out0,out1,out2,out3,ireg) \
2011+ { \
2012+ v128_t l = wasm_u16x8_extend_low_u8x16 ( ireg ); \
2013+ v128_t h = wasm_u16x8_extend_high_u8x16( ireg ); \
2014+ out0 = wasm_u32x4_extend_low_u16x8 ( l ); \
2015+ out1 = wasm_u32x4_extend_high_u16x8( l ); \
2016+ out2 = wasm_u32x4_extend_low_u16x8 ( h ); \
2017+ out3 = wasm_u32x4_extend_high_u16x8( h ); \
2018+ }
2019+
2020+ #define stbir__simdi_expand_u8_to_1u32(out,ireg) \
2021+ { \
2022+ v128_t tmp = wasm_u16x8_extend_low_u8x16(ireg); \
2023+ out = wasm_u32x4_extend_low_u16x8(tmp); \
2024+ }
2025+
2026+ #define stbir__simdi_expand_u16_to_u32(out0,out1,ireg) \
2027+ { \
2028+ out0 = wasm_u32x4_extend_low_u16x8 ( ireg ); \
2029+ out1 = wasm_u32x4_extend_high_u16x8( ireg ); \
2030+ }
2031+
2032+ #define stbir__simdf_convert_float_to_i32( i, f ) (i) = wasm_i32x4_trunc_sat_f32x4(f)
2033+ #define stbir__simdf_convert_float_to_int( f ) wasm_i32x4_extract_lane(wasm_i32x4_trunc_sat_f32x4(f), 0)
2034+ #define stbir__simdi_to_int( i ) wasm_i32x4_extract_lane(i, 0)
2035+ #define stbir__simdf_convert_float_to_uint8( f ) ((unsigned char)wasm_i32x4_extract_lane(wasm_i32x4_trunc_sat_f32x4(wasm_f32x4_max(wasm_f32x4_min(f,STBIR_max_uint8_as_float),wasm_f32x4_const_splat(0))), 0))
2036+ #define stbir__simdf_convert_float_to_short( f ) ((unsigned short)wasm_i32x4_extract_lane(wasm_i32x4_trunc_sat_f32x4(wasm_f32x4_max(wasm_f32x4_min(f,STBIR_max_uint16_as_float),wasm_f32x4_const_splat(0))), 0))
2037+ #define stbir__simdi_convert_i32_to_float(out, ireg) (out) = wasm_f32x4_convert_i32x4(ireg)
2038+ #define stbir__simdf_add( out, reg0, reg1 ) (out) = wasm_f32x4_add( reg0, reg1 )
2039+ #define stbir__simdf_mult( out, reg0, reg1 ) (out) = wasm_f32x4_mul( reg0, reg1 )
2040+ #define stbir__simdf_mult_mem( out, reg, ptr ) (out) = wasm_f32x4_mul( reg, wasm_v128_load( (void const*)(ptr) ) )
2041+ #define stbir__simdf_mult1_mem( out, reg, ptr ) (out) = wasm_f32x4_mul( reg, wasm_v128_load32_splat( (void const*)(ptr) ) )
2042+ #define stbir__simdf_add_mem( out, reg, ptr ) (out) = wasm_f32x4_add( reg, wasm_v128_load( (void const*)(ptr) ) )
2043+ #define stbir__simdf_add1_mem( out, reg, ptr ) (out) = wasm_f32x4_add( reg, wasm_v128_load32_splat( (void const*)(ptr) ) )
2044+
2045+ #define stbir__simdf_madd( out, add, mul1, mul2 ) (out) = wasm_f32x4_add( add, wasm_f32x4_mul( mul1, mul2 ) )
2046+ #define stbir__simdf_madd1( out, add, mul1, mul2 ) (out) = wasm_f32x4_add( add, wasm_f32x4_mul( mul1, mul2 ) )
2047+ #define stbir__simdf_madd_mem( out, add, mul, ptr ) (out) = wasm_f32x4_add( add, wasm_f32x4_mul( mul, wasm_v128_load( (void const*)(ptr) ) ) )
2048+ #define stbir__simdf_madd1_mem( out, add, mul, ptr ) (out) = wasm_f32x4_add( add, wasm_f32x4_mul( mul, wasm_v128_load32_splat( (void const*)(ptr) ) ) )
2049+
2050+ #define stbir__simdf_add1( out, reg0, reg1 ) (out) = wasm_f32x4_add( reg0, reg1 )
2051+ #define stbir__simdf_mult1( out, reg0, reg1 ) (out) = wasm_f32x4_mul( reg0, reg1 )
2052+
2053+ #define stbir__simdf_and( out, reg0, reg1 ) (out) = wasm_v128_and( reg0, reg1 )
2054+ #define stbir__simdf_or( out, reg0, reg1 ) (out) = wasm_v128_or( reg0, reg1 )
2055+
2056+ #define stbir__simdf_min( out, reg0, reg1 ) (out) = wasm_f32x4_min( reg0, reg1 )
2057+ #define stbir__simdf_max( out, reg0, reg1 ) (out) = wasm_f32x4_max( reg0, reg1 )
2058+ #define stbir__simdf_min1( out, reg0, reg1 ) (out) = wasm_f32x4_min( reg0, reg1 )
2059+ #define stbir__simdf_max1( out, reg0, reg1 ) (out) = wasm_f32x4_max( reg0, reg1 )
2060+
2061+ #define stbir__simdf_0123ABCDto3ABx( out, reg0, reg1 ) (out) = wasm_i32x4_shuffle( reg0, reg1, 3, 4, 5, -1 )
2062+ #define stbir__simdf_0123ABCDto23Ax( out, reg0, reg1 ) (out) = wasm_i32x4_shuffle( reg0, reg1, 2, 3, 4, -1 )
2063+
2064+ #define stbir__simdf_aaa1(out,alp,ones) (out) = wasm_i32x4_shuffle(alp, ones, 3, 3, 3, 4)
2065+ #define stbir__simdf_1aaa(out,alp,ones) (out) = wasm_i32x4_shuffle(alp, ones, 4, 0, 0, 0)
2066+ #define stbir__simdf_a1a1(out,alp,ones) (out) = wasm_i32x4_shuffle(alp, ones, 1, 4, 3, 4)
2067+ #define stbir__simdf_1a1a(out,alp,ones) (out) = wasm_i32x4_shuffle(alp, ones, 4, 0, 4, 2)
2068+
2069+ #define stbir__simdf_swiz( reg, one, two, three, four ) wasm_i32x4_shuffle(reg, reg, one, two, three, four)
2070+
2071+ #define stbir__simdi_and( out, reg0, reg1 ) (out) = wasm_v128_and( reg0, reg1 )
2072+ #define stbir__simdi_or( out, reg0, reg1 ) (out) = wasm_v128_or( reg0, reg1 )
2073+ #define stbir__simdi_16madd( out, reg0, reg1 ) (out) = wasm_i32x4_dot_i16x8( reg0, reg1 )
2074+
2075+ #define stbir__simdf_pack_to_8bytes(out,aa,bb) \
2076+ { \
2077+ v128_t af = wasm_f32x4_max( wasm_f32x4_min(aa, STBIR_max_uint8_as_float), wasm_f32x4_const_splat(0) ); \
2078+ v128_t bf = wasm_f32x4_max( wasm_f32x4_min(bb, STBIR_max_uint8_as_float), wasm_f32x4_const_splat(0) ); \
2079+ v128_t ai = wasm_i32x4_trunc_sat_f32x4( af ); \
2080+ v128_t bi = wasm_i32x4_trunc_sat_f32x4( bf ); \
2081+ v128_t out16 = wasm_i16x8_narrow_i32x4( ai, bi ); \
2082+ out = wasm_u8x16_narrow_i16x8( out16, out16 ); \
2083+ }
2084+
2085+ #define stbir__simdf_pack_to_8words(out,aa,bb) \
2086+ { \
2087+ v128_t af = wasm_f32x4_max( wasm_f32x4_min(aa, STBIR_max_uint16_as_float), wasm_f32x4_const_splat(0)); \
2088+ v128_t bf = wasm_f32x4_max( wasm_f32x4_min(bb, STBIR_max_uint16_as_float), wasm_f32x4_const_splat(0)); \
2089+ v128_t ai = wasm_i32x4_trunc_sat_f32x4( af ); \
2090+ v128_t bi = wasm_i32x4_trunc_sat_f32x4( bf ); \
2091+ out = wasm_u16x8_narrow_i32x4( ai, bi ); \
2092+ }
2093+
2094+ #define stbir__interleave_pack_and_store_16_u8( ptr, r0, r1, r2, r3 ) \
2095+ { \
2096+ v128_t tmp0 = wasm_i16x8_narrow_i32x4(r0, r1); \
2097+ v128_t tmp1 = wasm_i16x8_narrow_i32x4(r2, r3); \
2098+ v128_t tmp = wasm_u8x16_narrow_i16x8(tmp0, tmp1); \
2099+ tmp = wasm_i8x16_shuffle(tmp, tmp, 0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); \
2100+ wasm_v128_store( (void*)(ptr), tmp); \
2101+ }
2102+
2103+ #define stbir__simdf_load4_transposed( o0, o1, o2, o3, ptr ) \
2104+ { \
2105+ v128_t t0 = wasm_v128_load( ptr ); \
2106+ v128_t t1 = wasm_v128_load( ptr+4 ); \
2107+ v128_t t2 = wasm_v128_load( ptr+8 ); \
2108+ v128_t t3 = wasm_v128_load( ptr+12 ); \
2109+ v128_t s0 = wasm_i32x4_shuffle(t0, t1, 0, 4, 2, 6); \
2110+ v128_t s1 = wasm_i32x4_shuffle(t0, t1, 1, 5, 3, 7); \
2111+ v128_t s2 = wasm_i32x4_shuffle(t2, t3, 0, 4, 2, 6); \
2112+ v128_t s3 = wasm_i32x4_shuffle(t2, t3, 1, 5, 3, 7); \
2113+ o0 = wasm_i32x4_shuffle(s0, s2, 0, 1, 4, 5); \
2114+ o1 = wasm_i32x4_shuffle(s1, s3, 0, 1, 4, 5); \
2115+ o2 = wasm_i32x4_shuffle(s0, s2, 2, 3, 6, 7); \
2116+ o3 = wasm_i32x4_shuffle(s1, s3, 2, 3, 6, 7); \
2117+ }
2118+
2119+ #define stbir__simdi_32shr( out, reg, imm ) out = wasm_u32x4_shr( reg, imm )
2120+
2121+ typedef float stbir__f32x4 __attribute__((__vector_size__(16), __aligned__(16)));
2122+ #define STBIR__SIMDF_CONST(var, x) stbir__simdf var = (v128_t)(stbir__f32x4){ x, x, x, x }
2123+ #define STBIR__SIMDI_CONST(var, x) stbir__simdi var = { x, x, x, x }
2124+ #define STBIR__CONSTF(var) (var)
2125+ #define STBIR__CONSTI(var) (var)
2126+
2127+ #ifdef STBIR_FLOORF
2128+ #undef STBIR_FLOORF
2129+ #endif
2130+ #define STBIR_FLOORF stbir_simd_floorf
2131+ static stbir__inline float stbir_simd_floorf(float x)
2132+ {
2133+ return wasm_f32x4_extract_lane( wasm_f32x4_floor( wasm_f32x4_splat(x) ), 0);
2134+ }
2135+
2136+ #ifdef STBIR_CEILF
2137+ #undef STBIR_CEILF
2138+ #endif
2139+ #define STBIR_CEILF stbir_simd_ceilf
2140+ static stbir__inline float stbir_simd_ceilf(float x)
2141+ {
2142+ return wasm_f32x4_extract_lane( wasm_f32x4_ceil( wasm_f32x4_splat(x) ), 0);
2143+ }
2144+
2145+ #define STBIR_SIMD
2146+
2147+#endif // SSE2/NEON/WASM
2148+
2149+#endif // NO SIMD
2150+
2151+#ifdef STBIR_SIMD8
2152+ #define stbir__simdfX stbir__simdf8
2153+ #define stbir__simdiX stbir__simdi8
2154+ #define stbir__simdfX_load stbir__simdf8_load
2155+ #define stbir__simdiX_load stbir__simdi8_load
2156+ #define stbir__simdfX_mult stbir__simdf8_mult
2157+ #define stbir__simdfX_add_mem stbir__simdf8_add_mem
2158+ #define stbir__simdfX_madd_mem stbir__simdf8_madd_mem
2159+ #define stbir__simdfX_store stbir__simdf8_store
2160+ #define stbir__simdiX_store stbir__simdi8_store
2161+ #define stbir__simdf_frepX stbir__simdf8_frep8
2162+ #define stbir__simdfX_madd stbir__simdf8_madd
2163+ #define stbir__simdfX_min stbir__simdf8_min
2164+ #define stbir__simdfX_max stbir__simdf8_max
2165+ #define stbir__simdfX_aaa1 stbir__simdf8_aaa1
2166+ #define stbir__simdfX_1aaa stbir__simdf8_1aaa
2167+ #define stbir__simdfX_a1a1 stbir__simdf8_a1a1
2168+ #define stbir__simdfX_1a1a stbir__simdf8_1a1a
2169+ #define stbir__simdfX_convert_float_to_i32 stbir__simdf8_convert_float_to_i32
2170+ #define stbir__simdfX_pack_to_words stbir__simdf8_pack_to_16words
2171+ #define stbir__simdfX_zero stbir__simdf8_zero
2172+ #define STBIR_onesX STBIR_ones8
2173+ #define STBIR_max_uint8_as_floatX STBIR_max_uint8_as_float8
2174+ #define STBIR_max_uint16_as_floatX STBIR_max_uint16_as_float8
2175+ #define STBIR_simd_point5X STBIR_simd_point58
2176+ #define stbir__simdfX_float_count 8
2177+ #define stbir__simdfX_0123to1230 stbir__simdf8_0123to12301230
2178+ #define stbir__simdfX_0123to2103 stbir__simdf8_0123to21032103
2179+ static const stbir__simdf8 STBIR_max_uint16_as_float_inverted8 = { stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted };
2180+ static const stbir__simdf8 STBIR_max_uint8_as_float_inverted8 = { stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted };
2181+ static const stbir__simdf8 STBIR_ones8 = { 1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0 };
2182+ static const stbir__simdf8 STBIR_simd_point58 = { 0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5 };
2183+ static const stbir__simdf8 STBIR_max_uint8_as_float8 = { stbir__max_uint8_as_float,stbir__max_uint8_as_float,stbir__max_uint8_as_float,stbir__max_uint8_as_float, stbir__max_uint8_as_float,stbir__max_uint8_as_float,stbir__max_uint8_as_float,stbir__max_uint8_as_float };
2184+ static const stbir__simdf8 STBIR_max_uint16_as_float8 = { stbir__max_uint16_as_float,stbir__max_uint16_as_float,stbir__max_uint16_as_float,stbir__max_uint16_as_float, stbir__max_uint16_as_float,stbir__max_uint16_as_float,stbir__max_uint16_as_float,stbir__max_uint16_as_float };
2185+#else
2186+ #define stbir__simdfX stbir__simdf
2187+ #define stbir__simdiX stbir__simdi
2188+ #define stbir__simdfX_load stbir__simdf_load
2189+ #define stbir__simdiX_load stbir__simdi_load
2190+ #define stbir__simdfX_mult stbir__simdf_mult
2191+ #define stbir__simdfX_add_mem stbir__simdf_add_mem
2192+ #define stbir__simdfX_madd_mem stbir__simdf_madd_mem
2193+ #define stbir__simdfX_store stbir__simdf_store
2194+ #define stbir__simdiX_store stbir__simdi_store
2195+ #define stbir__simdf_frepX stbir__simdf_frep4
2196+ #define stbir__simdfX_madd stbir__simdf_madd
2197+ #define stbir__simdfX_min stbir__simdf_min
2198+ #define stbir__simdfX_max stbir__simdf_max
2199+ #define stbir__simdfX_aaa1 stbir__simdf_aaa1
2200+ #define stbir__simdfX_1aaa stbir__simdf_1aaa
2201+ #define stbir__simdfX_a1a1 stbir__simdf_a1a1
2202+ #define stbir__simdfX_1a1a stbir__simdf_1a1a
2203+ #define stbir__simdfX_convert_float_to_i32 stbir__simdf_convert_float_to_i32
2204+ #define stbir__simdfX_pack_to_words stbir__simdf_pack_to_8words
2205+ #define stbir__simdfX_zero stbir__simdf_zero
2206+ #define STBIR_onesX STBIR__CONSTF(STBIR_ones)
2207+ #define STBIR_simd_point5X STBIR__CONSTF(STBIR_simd_point5)
2208+ #define STBIR_max_uint8_as_floatX STBIR__CONSTF(STBIR_max_uint8_as_float)
2209+ #define STBIR_max_uint16_as_floatX STBIR__CONSTF(STBIR_max_uint16_as_float)
2210+ #define stbir__simdfX_float_count 4
2211+ #define stbir__if_simdf8_cast_to_simdf4( val ) ( val )
2212+ #define stbir__simdfX_0123to1230 stbir__simdf_0123to1230
2213+ #define stbir__simdfX_0123to2103 stbir__simdf_0123to2103
2214+#endif
2215+
2216+
2217+#if defined(STBIR_NEON) && !defined(_M_ARM) && !defined(__arm__)
2218+
2219+ #if defined( _MSC_VER ) && !defined(__clang__)
2220+ typedef __int16 stbir__FP16;
2221+ #else
2222+ typedef float16_t stbir__FP16;
2223+ #endif
2224+
2225+#else // no NEON, or 32-bit ARM for MSVC
2226+
2227+ typedef union stbir__FP16
2228+ {
2229+ unsigned short u;
2230+ } stbir__FP16;
2231+
2232+#endif
2233+
2234+#if (!defined(STBIR_NEON) && !defined(STBIR_FP16C)) || (defined(STBIR_NEON) && defined(_M_ARM)) || (defined(STBIR_NEON) && defined(__arm__))
2235+
2236+ // Fabian's half float routines, see: https://gist.github.com/rygorous/2156668
2237+
2238+ static stbir__inline float stbir__half_to_float( stbir__FP16 h )
2239+ {
2240+ static const stbir__FP32 magic = { (254 - 15) << 23 };
2241+ static const stbir__FP32 was_infnan = { (127 + 16) << 23 };
2242+ stbir__FP32 o;
2243+
2244+ o.u = (h.u & 0x7fff) << 13; // exponent/mantissa bits
2245+ o.f *= magic.f; // exponent adjust
2246+ if (o.f >= was_infnan.f) // make sure Inf/NaN survive
2247+ o.u |= 255 << 23;
2248+ o.u |= (h.u & 0x8000) << 16; // sign bit
2249+ return o.f;
2250+ }
2251+
2252+ static stbir__inline stbir__FP16 stbir__float_to_half(float val)
2253+ {
2254+ stbir__FP32 f32infty = { 255 << 23 };
2255+ stbir__FP32 f16max = { (127 + 16) << 23 };
2256+ stbir__FP32 denorm_magic = { ((127 - 15) + (23 - 10) + 1) << 23 };
2257+ unsigned int sign_mask = 0x80000000u;
2258+ stbir__FP16 o = { 0 };
2259+ stbir__FP32 f;
2260+ unsigned int sign;
2261+
2262+ f.f = val;
2263+ sign = f.u & sign_mask;
2264+ f.u ^= sign;
2265+
2266+ if (f.u >= f16max.u) // result is Inf or NaN (all exponent bits set)
2267+ o.u = (f.u > f32infty.u) ? 0x7e00 : 0x7c00; // NaN->qNaN and Inf->Inf
2268+ else // (De)normalized number or zero
2269+ {
2270+ if (f.u < (113 << 23)) // resulting FP16 is subnormal or zero
2271+ {
2272+ // use a magic value to align our 10 mantissa bits at the bottom of
2273+ // the float. as long as FP addition is round-to-nearest-even this
2274+ // just works.
2275+ f.f += denorm_magic.f;
2276+ // and one integer subtract of the bias later, we have our final float!
2277+ o.u = (unsigned short) ( f.u - denorm_magic.u );
2278+ }
2279+ else
2280+ {
2281+ unsigned int mant_odd = (f.u >> 13) & 1; // resulting mantissa is odd
2282+ // update exponent, rounding bias part 1
2283+ f.u = f.u + ((15u - 127) << 23) + 0xfff;
2284+ // rounding bias part 2
2285+ f.u += mant_odd;
2286+ // take the bits!
2287+ o.u = (unsigned short) ( f.u >> 13 );
2288+ }
2289+ }
2290+
2291+ o.u |= sign >> 16;
2292+ return o;
2293+ }
2294+
2295+#endif
2296+
2297+
2298+#if defined(STBIR_FP16C)
2299+
2300+ #include <immintrin.h>
2301+
2302+ static stbir__inline void stbir__half_to_float_SIMD(float * output, stbir__FP16 const * input)
2303+ {
2304+ _mm256_storeu_ps( (float*)output, _mm256_cvtph_ps( _mm_loadu_si128( (__m128i const* )input ) ) );
2305+ }
2306+
2307+ static stbir__inline void stbir__float_to_half_SIMD(stbir__FP16 * output, float const * input)
2308+ {
2309+ _mm_storeu_si128( (__m128i*)output, _mm256_cvtps_ph( _mm256_loadu_ps( input ), 0 ) );
2310+ }
2311+
2312+ static stbir__inline float stbir__half_to_float( stbir__FP16 h )
2313+ {
2314+ return _mm_cvtss_f32( _mm_cvtph_ps( _mm_cvtsi32_si128( (int)h.u ) ) );
2315+ }
2316+
2317+ static stbir__inline stbir__FP16 stbir__float_to_half( float f )
2318+ {
2319+ stbir__FP16 h;
2320+ h.u = (unsigned short) _mm_cvtsi128_si32( _mm_cvtps_ph( _mm_set_ss( f ), 0 ) );
2321+ return h;
2322+ }
2323+
2324+#elif defined(STBIR_SSE2)
2325+
2326+ // Fabian's half float routines, see: https://gist.github.com/rygorous/2156668
2327+ stbir__inline static void stbir__half_to_float_SIMD(float * output, void const * input)
2328+ {
2329+ static const STBIR__SIMDI_CONST(mask_nosign, 0x7fff);
2330+ static const STBIR__SIMDI_CONST(smallest_normal, 0x0400);
2331+ static const STBIR__SIMDI_CONST(infinity, 0x7c00);
2332+ static const STBIR__SIMDI_CONST(expadjust_normal, (127 - 15) << 23);
2333+ static const STBIR__SIMDI_CONST(magic_denorm, 113 << 23);
2334+
2335+ __m128i i = _mm_loadu_si128 ( (__m128i const*)(input) );
2336+ __m128i h = _mm_unpacklo_epi16 ( i, _mm_setzero_si128() );
2337+ __m128i mnosign = STBIR__CONSTI(mask_nosign);
2338+ __m128i eadjust = STBIR__CONSTI(expadjust_normal);
2339+ __m128i smallest = STBIR__CONSTI(smallest_normal);
2340+ __m128i infty = STBIR__CONSTI(infinity);
2341+ __m128i expmant = _mm_and_si128(mnosign, h);
2342+ __m128i justsign = _mm_xor_si128(h, expmant);
2343+ __m128i b_notinfnan = _mm_cmpgt_epi32(infty, expmant);
2344+ __m128i b_isdenorm = _mm_cmpgt_epi32(smallest, expmant);
2345+ __m128i shifted = _mm_slli_epi32(expmant, 13);
2346+ __m128i adj_infnan = _mm_andnot_si128(b_notinfnan, eadjust);
2347+ __m128i adjusted = _mm_add_epi32(eadjust, shifted);
2348+ __m128i den1 = _mm_add_epi32(shifted, STBIR__CONSTI(magic_denorm));
2349+ __m128i adjusted2 = _mm_add_epi32(adjusted, adj_infnan);
2350+ __m128 den2 = _mm_sub_ps(_mm_castsi128_ps(den1), *(const __m128 *)&magic_denorm);
2351+ __m128 adjusted3 = _mm_and_ps(den2, _mm_castsi128_ps(b_isdenorm));
2352+ __m128 adjusted4 = _mm_andnot_ps(_mm_castsi128_ps(b_isdenorm), _mm_castsi128_ps(adjusted2));
2353+ __m128 adjusted5 = _mm_or_ps(adjusted3, adjusted4);
2354+ __m128i sign = _mm_slli_epi32(justsign, 16);
2355+ __m128 final = _mm_or_ps(adjusted5, _mm_castsi128_ps(sign));
2356+ stbir__simdf_store( output + 0, final );
2357+
2358+ h = _mm_unpackhi_epi16 ( i, _mm_setzero_si128() );
2359+ expmant = _mm_and_si128(mnosign, h);
2360+ justsign = _mm_xor_si128(h, expmant);
2361+ b_notinfnan = _mm_cmpgt_epi32(infty, expmant);
2362+ b_isdenorm = _mm_cmpgt_epi32(smallest, expmant);
2363+ shifted = _mm_slli_epi32(expmant, 13);
2364+ adj_infnan = _mm_andnot_si128(b_notinfnan, eadjust);
2365+ adjusted = _mm_add_epi32(eadjust, shifted);
2366+ den1 = _mm_add_epi32(shifted, STBIR__CONSTI(magic_denorm));
2367+ adjusted2 = _mm_add_epi32(adjusted, adj_infnan);
2368+ den2 = _mm_sub_ps(_mm_castsi128_ps(den1), *(const __m128 *)&magic_denorm);
2369+ adjusted3 = _mm_and_ps(den2, _mm_castsi128_ps(b_isdenorm));
2370+ adjusted4 = _mm_andnot_ps(_mm_castsi128_ps(b_isdenorm), _mm_castsi128_ps(adjusted2));
2371+ adjusted5 = _mm_or_ps(adjusted3, adjusted4);
2372+ sign = _mm_slli_epi32(justsign, 16);
2373+ final = _mm_or_ps(adjusted5, _mm_castsi128_ps(sign));
2374+ stbir__simdf_store( output + 4, final );
2375+
2376+ // ~38 SSE2 ops for 8 values
2377+ }
2378+
2379+ // Fabian's round-to-nearest-even float to half
2380+ // ~48 SSE2 ops for 8 output
2381+ stbir__inline static void stbir__float_to_half_SIMD(void * output, float const * input)
2382+ {
2383+ static const STBIR__SIMDI_CONST(mask_sign, 0x80000000u);
2384+ static const STBIR__SIMDI_CONST(c_f16max, (127 + 16) << 23); // all FP32 values >=this round to +inf
2385+ static const STBIR__SIMDI_CONST(c_nanbit, 0x200);
2386+ static const STBIR__SIMDI_CONST(c_infty_as_fp16, 0x7c00);
2387+ static const STBIR__SIMDI_CONST(c_min_normal, (127 - 14) << 23); // smallest FP32 that yields a normalized FP16
2388+ static const STBIR__SIMDI_CONST(c_subnorm_magic, ((127 - 15) + (23 - 10) + 1) << 23);
2389+ static const STBIR__SIMDI_CONST(c_normal_bias, 0xfff - ((127 - 15) << 23)); // adjust exponent and add mantissa rounding
2390+
2391+ __m128 f = _mm_loadu_ps(input);
2392+ __m128 msign = _mm_castsi128_ps(STBIR__CONSTI(mask_sign));
2393+ __m128 justsign = _mm_and_ps(msign, f);
2394+ __m128 absf = _mm_xor_ps(f, justsign);
2395+ __m128i absf_int = _mm_castps_si128(absf); // the cast is "free" (extra bypass latency, but no thruput hit)
2396+ __m128i f16max = STBIR__CONSTI(c_f16max);
2397+ __m128 b_isnan = _mm_cmpunord_ps(absf, absf); // is this a NaN?
2398+ __m128i b_isregular = _mm_cmpgt_epi32(f16max, absf_int); // (sub)normalized or special?
2399+ __m128i nanbit = _mm_and_si128(_mm_castps_si128(b_isnan), STBIR__CONSTI(c_nanbit));
2400+ __m128i inf_or_nan = _mm_or_si128(nanbit, STBIR__CONSTI(c_infty_as_fp16)); // output for specials
2401+
2402+ __m128i min_normal = STBIR__CONSTI(c_min_normal);
2403+ __m128i b_issub = _mm_cmpgt_epi32(min_normal, absf_int);
2404+
2405+ // "result is subnormal" path
2406+ __m128 subnorm1 = _mm_add_ps(absf, _mm_castsi128_ps(STBIR__CONSTI(c_subnorm_magic))); // magic value to round output mantissa
2407+ __m128i subnorm2 = _mm_sub_epi32(_mm_castps_si128(subnorm1), STBIR__CONSTI(c_subnorm_magic)); // subtract out bias
2408+
2409+ // "result is normal" path
2410+ __m128i mantoddbit = _mm_slli_epi32(absf_int, 31 - 13); // shift bit 13 (mantissa LSB) to sign
2411+ __m128i mantodd = _mm_srai_epi32(mantoddbit, 31); // -1 if FP16 mantissa odd, else 0
2412+
2413+ __m128i round1 = _mm_add_epi32(absf_int, STBIR__CONSTI(c_normal_bias));
2414+ __m128i round2 = _mm_sub_epi32(round1, mantodd); // if mantissa LSB odd, bias towards rounding up (RTNE)
2415+ __m128i normal = _mm_srli_epi32(round2, 13); // rounded result
2416+
2417+ // combine the two non-specials
2418+ __m128i nonspecial = _mm_or_si128(_mm_and_si128(subnorm2, b_issub), _mm_andnot_si128(b_issub, normal));
2419+
2420+ // merge in specials as well
2421+ __m128i joined = _mm_or_si128(_mm_and_si128(nonspecial, b_isregular), _mm_andnot_si128(b_isregular, inf_or_nan));
2422+
2423+ __m128i sign_shift = _mm_srai_epi32(_mm_castps_si128(justsign), 16);
2424+ __m128i final2, final= _mm_or_si128(joined, sign_shift);
2425+
2426+ f = _mm_loadu_ps(input+4);
2427+ justsign = _mm_and_ps(msign, f);
2428+ absf = _mm_xor_ps(f, justsign);
2429+ absf_int = _mm_castps_si128(absf); // the cast is "free" (extra bypass latency, but no thruput hit)
2430+ b_isnan = _mm_cmpunord_ps(absf, absf); // is this a NaN?
2431+ b_isregular = _mm_cmpgt_epi32(f16max, absf_int); // (sub)normalized or special?
2432+ nanbit = _mm_and_si128(_mm_castps_si128(b_isnan), c_nanbit);
2433+ inf_or_nan = _mm_or_si128(nanbit, STBIR__CONSTI(c_infty_as_fp16)); // output for specials
2434+
2435+ b_issub = _mm_cmpgt_epi32(min_normal, absf_int);
2436+
2437+ // "result is subnormal" path
2438+ subnorm1 = _mm_add_ps(absf, _mm_castsi128_ps(STBIR__CONSTI(c_subnorm_magic))); // magic value to round output mantissa
2439+ subnorm2 = _mm_sub_epi32(_mm_castps_si128(subnorm1), STBIR__CONSTI(c_subnorm_magic)); // subtract out bias
2440+
2441+ // "result is normal" path
2442+ mantoddbit = _mm_slli_epi32(absf_int, 31 - 13); // shift bit 13 (mantissa LSB) to sign
2443+ mantodd = _mm_srai_epi32(mantoddbit, 31); // -1 if FP16 mantissa odd, else 0
2444+
2445+ round1 = _mm_add_epi32(absf_int, STBIR__CONSTI(c_normal_bias));
2446+ round2 = _mm_sub_epi32(round1, mantodd); // if mantissa LSB odd, bias towards rounding up (RTNE)
2447+ normal = _mm_srli_epi32(round2, 13); // rounded result
2448+
2449+ // combine the two non-specials
2450+ nonspecial = _mm_or_si128(_mm_and_si128(subnorm2, b_issub), _mm_andnot_si128(b_issub, normal));
2451+
2452+ // merge in specials as well
2453+ joined = _mm_or_si128(_mm_and_si128(nonspecial, b_isregular), _mm_andnot_si128(b_isregular, inf_or_nan));
2454+
2455+ sign_shift = _mm_srai_epi32(_mm_castps_si128(justsign), 16);
2456+ final2 = _mm_or_si128(joined, sign_shift);
2457+ final = _mm_packs_epi32(final, final2);
2458+ stbir__simdi_store( output,final );
2459+ }
2460+
2461+#elif defined(STBIR_NEON) && defined(_MSC_VER) && defined(_M_ARM64) && !defined(__clang__) // 64-bit ARM on MSVC (not clang)
2462+
2463+ static stbir__inline void stbir__half_to_float_SIMD(float * output, stbir__FP16 const * input)
2464+ {
2465+ float16x4_t in0 = vld1_f16(input + 0);
2466+ float16x4_t in1 = vld1_f16(input + 4);
2467+ vst1q_f32(output + 0, vcvt_f32_f16(in0));
2468+ vst1q_f32(output + 4, vcvt_f32_f16(in1));
2469+ }
2470+
2471+ static stbir__inline void stbir__float_to_half_SIMD(stbir__FP16 * output, float const * input)
2472+ {
2473+ float16x4_t out0 = vcvt_f16_f32(vld1q_f32(input + 0));
2474+ float16x4_t out1 = vcvt_f16_f32(vld1q_f32(input + 4));
2475+ vst1_f16(output+0, out0);
2476+ vst1_f16(output+4, out1);
2477+ }
2478+
2479+ static stbir__inline float stbir__half_to_float( stbir__FP16 h )
2480+ {
2481+ return vgetq_lane_f32(vcvt_f32_f16(vld1_dup_f16(&h)), 0);
2482+ }
2483+
2484+ static stbir__inline stbir__FP16 stbir__float_to_half( float f )
2485+ {
2486+ return vget_lane_f16(vcvt_f16_f32(vdupq_n_f32(f)), 0).n16_u16[0];
2487+ }
2488+
2489+#elif defined(STBIR_NEON) && ( defined( _M_ARM64 ) || defined( __aarch64__ ) || defined( __arm64__ ) ) // 64-bit ARM
2490+
2491+ static stbir__inline void stbir__half_to_float_SIMD(float * output, stbir__FP16 const * input)
2492+ {
2493+ float16x8_t in = vld1q_f16(input);
2494+ vst1q_f32(output + 0, vcvt_f32_f16(vget_low_f16(in)));
2495+ vst1q_f32(output + 4, vcvt_f32_f16(vget_high_f16(in)));
2496+ }
2497+
2498+ static stbir__inline void stbir__float_to_half_SIMD(stbir__FP16 * output, float const * input)
2499+ {
2500+ float16x4_t out0 = vcvt_f16_f32(vld1q_f32(input + 0));
2501+ float16x4_t out1 = vcvt_f16_f32(vld1q_f32(input + 4));
2502+ vst1q_f16(output, vcombine_f16(out0, out1));
2503+ }
2504+
2505+ static stbir__inline float stbir__half_to_float( stbir__FP16 h )
2506+ {
2507+ return vgetq_lane_f32(vcvt_f32_f16(vdup_n_f16(h)), 0);
2508+ }
2509+
2510+ static stbir__inline stbir__FP16 stbir__float_to_half( float f )
2511+ {
2512+ return vget_lane_f16(vcvt_f16_f32(vdupq_n_f32(f)), 0);
2513+ }
2514+
2515+#elif defined(STBIR_WASM) || (defined(STBIR_NEON) && (defined(_MSC_VER) || defined(_M_ARM) || defined(__arm__))) // WASM or 32-bit ARM on MSVC/clang
2516+
2517+ static stbir__inline void stbir__half_to_float_SIMD(float * output, stbir__FP16 const * input)
2518+ {
2519+ for (int i=0; i<8; i++)
2520+ {
2521+ output[i] = stbir__half_to_float(input[i]);
2522+ }
2523+ }
2524+ static stbir__inline void stbir__float_to_half_SIMD(stbir__FP16 * output, float const * input)
2525+ {
2526+ for (int i=0; i<8; i++)
2527+ {
2528+ output[i] = stbir__float_to_half(input[i]);
2529+ }
2530+ }
2531+
2532+#endif
2533+
2534+
2535+#ifdef STBIR_SIMD
2536+
2537+#define stbir__simdf_0123to3333( out, reg ) (out) = stbir__simdf_swiz( reg, 3,3,3,3 )
2538+#define stbir__simdf_0123to2222( out, reg ) (out) = stbir__simdf_swiz( reg, 2,2,2,2 )
2539+#define stbir__simdf_0123to1111( out, reg ) (out) = stbir__simdf_swiz( reg, 1,1,1,1 )
2540+#define stbir__simdf_0123to0000( out, reg ) (out) = stbir__simdf_swiz( reg, 0,0,0,0 )
2541+#define stbir__simdf_0123to0003( out, reg ) (out) = stbir__simdf_swiz( reg, 0,0,0,3 )
2542+#define stbir__simdf_0123to0001( out, reg ) (out) = stbir__simdf_swiz( reg, 0,0,0,1 )
2543+#define stbir__simdf_0123to1122( out, reg ) (out) = stbir__simdf_swiz( reg, 1,1,2,2 )
2544+#define stbir__simdf_0123to2333( out, reg ) (out) = stbir__simdf_swiz( reg, 2,3,3,3 )
2545+#define stbir__simdf_0123to0023( out, reg ) (out) = stbir__simdf_swiz( reg, 0,0,2,3 )
2546+#define stbir__simdf_0123to1230( out, reg ) (out) = stbir__simdf_swiz( reg, 1,2,3,0 )
2547+#define stbir__simdf_0123to2103( out, reg ) (out) = stbir__simdf_swiz( reg, 2,1,0,3 )
2548+#define stbir__simdf_0123to3210( out, reg ) (out) = stbir__simdf_swiz( reg, 3,2,1,0 )
2549+#define stbir__simdf_0123to2301( out, reg ) (out) = stbir__simdf_swiz( reg, 2,3,0,1 )
2550+#define stbir__simdf_0123to3012( out, reg ) (out) = stbir__simdf_swiz( reg, 3,0,1,2 )
2551+#define stbir__simdf_0123to0011( out, reg ) (out) = stbir__simdf_swiz( reg, 0,0,1,1 )
2552+#define stbir__simdf_0123to1100( out, reg ) (out) = stbir__simdf_swiz( reg, 1,1,0,0 )
2553+#define stbir__simdf_0123to2233( out, reg ) (out) = stbir__simdf_swiz( reg, 2,2,3,3 )
2554+#define stbir__simdf_0123to1133( out, reg ) (out) = stbir__simdf_swiz( reg, 1,1,3,3 )
2555+#define stbir__simdf_0123to0022( out, reg ) (out) = stbir__simdf_swiz( reg, 0,0,2,2 )
2556+#define stbir__simdf_0123to1032( out, reg ) (out) = stbir__simdf_swiz( reg, 1,0,3,2 )
2557+
2558+typedef union stbir__simdi_u32
2559+{
2560+ stbir_uint32 m128i_u32[4];
2561+ int m128i_i32[4];
2562+ stbir__simdi m128i_i128;
2563+} stbir__simdi_u32;
2564+
2565+static const int STBIR_mask[9] = { 0,0,0,-1,-1,-1,0,0,0 };
2566+
2567+static const STBIR__SIMDF_CONST(STBIR_max_uint8_as_float, stbir__max_uint8_as_float);
2568+static const STBIR__SIMDF_CONST(STBIR_max_uint16_as_float, stbir__max_uint16_as_float);
2569+static const STBIR__SIMDF_CONST(STBIR_max_uint8_as_float_inverted, stbir__max_uint8_as_float_inverted);
2570+static const STBIR__SIMDF_CONST(STBIR_max_uint16_as_float_inverted, stbir__max_uint16_as_float_inverted);
2571+
2572+static const STBIR__SIMDF_CONST(STBIR_simd_point5, 0.5f);
2573+static const STBIR__SIMDF_CONST(STBIR_ones, 1.0f);
2574+static const STBIR__SIMDI_CONST(STBIR_almost_zero, (127 - 13) << 23);
2575+static const STBIR__SIMDI_CONST(STBIR_almost_one, 0x3f7fffff);
2576+static const STBIR__SIMDI_CONST(STBIR_mantissa_mask, 0xff);
2577+static const STBIR__SIMDI_CONST(STBIR_topscale, 0x02000000);
2578+
2579+// Basically, in simd mode, we unroll the proper amount, and we don't want
2580+// the non-simd remnant loops to be unroll because they only run a few times
2581+// Adding this switch saves about 5K on clang which is Captain Unroll the 3rd.
2582+#define STBIR_SIMD_STREAMOUT_PTR( star ) STBIR_STREAMOUT_PTR( star )
2583+#define STBIR_SIMD_NO_UNROLL(ptr) STBIR_NO_UNROLL(ptr)
2584+#define STBIR_SIMD_NO_UNROLL_LOOP_START STBIR_NO_UNROLL_LOOP_START
2585+#define STBIR_SIMD_NO_UNROLL_LOOP_START_INF_FOR STBIR_NO_UNROLL_LOOP_START_INF_FOR
2586+
2587+#ifdef STBIR_MEMCPY
2588+#undef STBIR_MEMCPY
2589+#endif
2590+#define STBIR_MEMCPY stbir_simd_memcpy
2591+
2592+// override normal use of memcpy with much simpler copy (faster and smaller with our sized copies)
2593+static void stbir_simd_memcpy( void * dest, void const * src, size_t bytes )
2594+{
2595+ char STBIR_SIMD_STREAMOUT_PTR (*) d = (char*) dest;
2596+ char STBIR_SIMD_STREAMOUT_PTR( * ) d_end = ((char*) dest) + bytes;
2597+ ptrdiff_t ofs_to_src = (char*)src - (char*)dest;
2598+
2599+ // check overlaps
2600+ STBIR_ASSERT( ( ( d >= ( (char*)src) + bytes ) ) || ( ( d + bytes ) <= (char*)src ) );
2601+
2602+ if ( bytes < (16*stbir__simdfX_float_count) )
2603+ {
2604+ if ( bytes < 16 )
2605+ {
2606+ if ( bytes )
2607+ {
2608+ STBIR_SIMD_NO_UNROLL_LOOP_START
2609+ do
2610+ {
2611+ STBIR_SIMD_NO_UNROLL(d);
2612+ d[ 0 ] = d[ ofs_to_src ];
2613+ ++d;
2614+ } while ( d < d_end );
2615+ }
2616+ }
2617+ else
2618+ {
2619+ stbir__simdf x;
2620+ // do one unaligned to get us aligned for the stream out below
2621+ stbir__simdf_load( x, ( d + ofs_to_src ) );
2622+ stbir__simdf_store( d, x );
2623+ d = (char*)( ( ( (size_t)d ) + 16 ) & ~15 );
2624+
2625+ STBIR_SIMD_NO_UNROLL_LOOP_START_INF_FOR
2626+ for(;;)
2627+ {
2628+ STBIR_SIMD_NO_UNROLL(d);
2629+
2630+ if ( d > ( d_end - 16 ) )
2631+ {
2632+ if ( d == d_end )
2633+ return;
2634+ d = d_end - 16;
2635+ }
2636+
2637+ stbir__simdf_load( x, ( d + ofs_to_src ) );
2638+ stbir__simdf_store( d, x );
2639+ d += 16;
2640+ }
2641+ }
2642+ }
2643+ else
2644+ {
2645+ stbir__simdfX x0,x1,x2,x3;
2646+
2647+ // do one unaligned to get us aligned for the stream out below
2648+ stbir__simdfX_load( x0, ( d + ofs_to_src ) + 0*stbir__simdfX_float_count );
2649+ stbir__simdfX_load( x1, ( d + ofs_to_src ) + 4*stbir__simdfX_float_count );
2650+ stbir__simdfX_load( x2, ( d + ofs_to_src ) + 8*stbir__simdfX_float_count );
2651+ stbir__simdfX_load( x3, ( d + ofs_to_src ) + 12*stbir__simdfX_float_count );
2652+ stbir__simdfX_store( d + 0*stbir__simdfX_float_count, x0 );
2653+ stbir__simdfX_store( d + 4*stbir__simdfX_float_count, x1 );
2654+ stbir__simdfX_store( d + 8*stbir__simdfX_float_count, x2 );
2655+ stbir__simdfX_store( d + 12*stbir__simdfX_float_count, x3 );
2656+ d = (char*)( ( ( (size_t)d ) + (16*stbir__simdfX_float_count) ) & ~((16*stbir__simdfX_float_count)-1) );
2657+
2658+ STBIR_SIMD_NO_UNROLL_LOOP_START_INF_FOR
2659+ for(;;)
2660+ {
2661+ STBIR_SIMD_NO_UNROLL(d);
2662+
2663+ if ( d > ( d_end - (16*stbir__simdfX_float_count) ) )
2664+ {
2665+ if ( d == d_end )
2666+ return;
2667+ d = d_end - (16*stbir__simdfX_float_count);
2668+ }
2669+
2670+ stbir__simdfX_load( x0, ( d + ofs_to_src ) + 0*stbir__simdfX_float_count );
2671+ stbir__simdfX_load( x1, ( d + ofs_to_src ) + 4*stbir__simdfX_float_count );
2672+ stbir__simdfX_load( x2, ( d + ofs_to_src ) + 8*stbir__simdfX_float_count );
2673+ stbir__simdfX_load( x3, ( d + ofs_to_src ) + 12*stbir__simdfX_float_count );
2674+ stbir__simdfX_store( d + 0*stbir__simdfX_float_count, x0 );
2675+ stbir__simdfX_store( d + 4*stbir__simdfX_float_count, x1 );
2676+ stbir__simdfX_store( d + 8*stbir__simdfX_float_count, x2 );
2677+ stbir__simdfX_store( d + 12*stbir__simdfX_float_count, x3 );
2678+ d += (16*stbir__simdfX_float_count);
2679+ }
2680+ }
2681+}
2682+
2683+// memcpy that is specically intentionally overlapping (src is smaller then dest, so can be
2684+// a normal forward copy, bytes is divisible by 4 and bytes is greater than or equal to
2685+// the diff between dest and src)
2686+static void stbir_overlapping_memcpy( void * dest, void const * src, size_t bytes )
2687+{
2688+ char STBIR_SIMD_STREAMOUT_PTR (*) sd = (char*) src;
2689+ char STBIR_SIMD_STREAMOUT_PTR( * ) s_end = ((char*) src) + bytes;
2690+ ptrdiff_t ofs_to_dest = (char*)dest - (char*)src;
2691+
2692+ if ( ofs_to_dest >= 16 ) // is the overlap more than 16 away?
2693+ {
2694+ char STBIR_SIMD_STREAMOUT_PTR( * ) s_end16 = ((char*) src) + (bytes&~15);
2695+ STBIR_SIMD_NO_UNROLL_LOOP_START
2696+ do
2697+ {
2698+ stbir__simdf x;
2699+ STBIR_SIMD_NO_UNROLL(sd);
2700+ stbir__simdf_load( x, sd );
2701+ stbir__simdf_store( ( sd + ofs_to_dest ), x );
2702+ sd += 16;
2703+ } while ( sd < s_end16 );
2704+
2705+ if ( sd == s_end )
2706+ return;
2707+ }
2708+
2709+ do
2710+ {
2711+ STBIR_SIMD_NO_UNROLL(sd);
2712+ *(int*)( sd + ofs_to_dest ) = *(int*) sd;
2713+ sd += 4;
2714+ } while ( sd < s_end );
2715+}
2716+
2717+#else // no SSE2
2718+
2719+// when in scalar mode, we let unrolling happen, so this macro just does the __restrict
2720+#define STBIR_SIMD_STREAMOUT_PTR( star ) STBIR_STREAMOUT_PTR( star )
2721+#define STBIR_SIMD_NO_UNROLL(ptr)
2722+#define STBIR_SIMD_NO_UNROLL_LOOP_START
2723+#define STBIR_SIMD_NO_UNROLL_LOOP_START_INF_FOR
2724+
2725+#endif // SSE2
2726+
2727+
2728+#ifdef STBIR_PROFILE
2729+
2730+#ifndef STBIR_PROFILE_FUNC
2731+
2732+#if defined(_x86_64) || defined( __x86_64__ ) || defined( _M_X64 ) || defined(__x86_64) || defined(__SSE2__) || defined(STBIR_SSE) || defined( _M_IX86_FP ) || defined(__i386) || defined( __i386__ ) || defined( _M_IX86 ) || defined( _X86_ )
2733+
2734+#ifdef _MSC_VER
2735+
2736+ STBIRDEF stbir_uint64 __rdtsc();
2737+ #define STBIR_PROFILE_FUNC() __rdtsc()
2738+
2739+#else // non msvc
2740+
2741+ static stbir__inline stbir_uint64 STBIR_PROFILE_FUNC()
2742+ {
2743+ stbir_uint32 lo, hi;
2744+ asm volatile ("rdtsc" : "=a" (lo), "=d" (hi) );
2745+ return ( ( (stbir_uint64) hi ) << 32 ) | ( (stbir_uint64) lo );
2746+ }
2747+
2748+#endif // msvc
2749+
2750+#elif defined( _M_ARM64 ) || defined( __aarch64__ ) || defined( __arm64__ ) || defined(__ARM_NEON__)
2751+
2752+#if defined( _MSC_VER ) && !defined(__clang__)
2753+
2754+ #define STBIR_PROFILE_FUNC() _ReadStatusReg(ARM64_CNTVCT)
2755+
2756+#else
2757+
2758+ static stbir__inline stbir_uint64 STBIR_PROFILE_FUNC()
2759+ {
2760+ stbir_uint64 tsc;
2761+ asm volatile("mrs %0, cntvct_el0" : "=r" (tsc));
2762+ return tsc;
2763+ }
2764+
2765+#endif
2766+
2767+#else // x64, arm
2768+
2769+#error Unknown platform for profiling.
2770+
2771+#endif // x64, arm
2772+
2773+#endif // STBIR_PROFILE_FUNC
2774+
2775+#define STBIR_ONLY_PROFILE_GET_SPLIT_INFO ,stbir__per_split_info * split_info
2776+#define STBIR_ONLY_PROFILE_SET_SPLIT_INFO ,split_info
2777+
2778+#define STBIR_ONLY_PROFILE_BUILD_GET_INFO ,stbir__info * profile_info
2779+#define STBIR_ONLY_PROFILE_BUILD_SET_INFO ,profile_info
2780+
2781+// super light-weight micro profiler
2782+#define STBIR_PROFILE_START_ll( info, wh ) { stbir_uint64 wh##thiszonetime = STBIR_PROFILE_FUNC(); stbir_uint64 * wh##save_parent_excluded_ptr = info->current_zone_excluded_ptr; stbir_uint64 wh##current_zone_excluded = 0; info->current_zone_excluded_ptr = &wh##current_zone_excluded;
2783+#define STBIR_PROFILE_END_ll( info, wh ) wh##thiszonetime = STBIR_PROFILE_FUNC() - wh##thiszonetime; info->profile.named.wh += wh##thiszonetime - wh##current_zone_excluded; *wh##save_parent_excluded_ptr += wh##thiszonetime; info->current_zone_excluded_ptr = wh##save_parent_excluded_ptr; }
2784+#define STBIR_PROFILE_FIRST_START_ll( info, wh ) { int i; info->current_zone_excluded_ptr = &info->profile.named.total; for(i=0;i<STBIR__ARRAY_SIZE(info->profile.array);i++) info->profile.array[i]=0; } STBIR_PROFILE_START_ll( info, wh );
2785+#define STBIR_PROFILE_CLEAR_EXTRAS_ll( info, num ) { int extra; for(extra=1;extra<(num);extra++) { int i; for(i=0;i<STBIR__ARRAY_SIZE((info)->profile.array);i++) (info)[extra].profile.array[i]=0; } }
2786+
2787+// for thread data
2788+#define STBIR_PROFILE_START( wh ) STBIR_PROFILE_START_ll( split_info, wh )
2789+#define STBIR_PROFILE_END( wh ) STBIR_PROFILE_END_ll( split_info, wh )
2790+#define STBIR_PROFILE_FIRST_START( wh ) STBIR_PROFILE_FIRST_START_ll( split_info, wh )
2791+#define STBIR_PROFILE_CLEAR_EXTRAS() STBIR_PROFILE_CLEAR_EXTRAS_ll( split_info, split_count )
2792+
2793+// for build data
2794+#define STBIR_PROFILE_BUILD_START( wh ) STBIR_PROFILE_START_ll( profile_info, wh )
2795+#define STBIR_PROFILE_BUILD_END( wh ) STBIR_PROFILE_END_ll( profile_info, wh )
2796+#define STBIR_PROFILE_BUILD_FIRST_START( wh ) STBIR_PROFILE_FIRST_START_ll( profile_info, wh )
2797+#define STBIR_PROFILE_BUILD_CLEAR( info ) { int i; for(i=0;i<STBIR__ARRAY_SIZE(info->profile.array);i++) info->profile.array[i]=0; }
2798+
2799+#else // no profile
2800+
2801+#define STBIR_ONLY_PROFILE_GET_SPLIT_INFO
2802+#define STBIR_ONLY_PROFILE_SET_SPLIT_INFO
2803+
2804+#define STBIR_ONLY_PROFILE_BUILD_GET_INFO
2805+#define STBIR_ONLY_PROFILE_BUILD_SET_INFO
2806+
2807+#define STBIR_PROFILE_START( wh )
2808+#define STBIR_PROFILE_END( wh )
2809+#define STBIR_PROFILE_FIRST_START( wh )
2810+#define STBIR_PROFILE_CLEAR_EXTRAS( )
2811+
2812+#define STBIR_PROFILE_BUILD_START( wh )
2813+#define STBIR_PROFILE_BUILD_END( wh )
2814+#define STBIR_PROFILE_BUILD_FIRST_START( wh )
2815+#define STBIR_PROFILE_BUILD_CLEAR( info )
2816+
2817+#endif // stbir_profile
2818+
2819+#ifndef STBIR_CEILF
2820+#include <math.h>
2821+#if _MSC_VER <= 1200 // support VC6 for Sean
2822+#define STBIR_CEILF(x) ((float)ceil((float)(x)))
2823+#define STBIR_FLOORF(x) ((float)floor((float)(x)))
2824+#else
2825+#define STBIR_CEILF(x) ceilf(x)
2826+#define STBIR_FLOORF(x) floorf(x)
2827+#endif
2828+#endif
2829+
2830+#ifndef STBIR_MEMCPY
2831+// For memcpy
2832+#include <string.h>
2833+#define STBIR_MEMCPY( dest, src, len ) memcpy( dest, src, len )
2834+#endif
2835+
2836+#ifndef STBIR_SIMD
2837+
2838+// memcpy that is specifically intentionally overlapping (src is smaller then dest, so can be
2839+// a normal forward copy, bytes is divisible by 4 and bytes is greater than or equal to
2840+// the diff between dest and src)
2841+static void stbir_overlapping_memcpy( void * dest, void const * src, size_t bytes )
2842+{
2843+ char STBIR_SIMD_STREAMOUT_PTR (*) sd = (char*) src;
2844+ char STBIR_SIMD_STREAMOUT_PTR( * ) s_end = ((char*) src) + bytes;
2845+ ptrdiff_t ofs_to_dest = (char*)dest - (char*)src;
2846+
2847+ if ( ofs_to_dest >= 8 ) // is the overlap more than 8 away
2848+ {
2849+ char STBIR_SIMD_STREAMOUT_PTR( * ) s_end8 = ((char*) src) + (bytes&~7);
2850+
2851+ if ( ( ( ((ptrdiff_t)dest)|((ptrdiff_t)src) ) & 7 ) == 0 ) // is it 8byte aligned?
2852+ {
2853+ STBIR_NO_UNROLL_LOOP_START
2854+ do
2855+ {
2856+ STBIR_NO_UNROLL(sd);
2857+ *(stbir_uint64*)( sd + ofs_to_dest ) = *(stbir_uint64*) sd;
2858+ sd += 8;
2859+ } while ( sd < s_end8 );
2860+ }
2861+ else
2862+ {
2863+ STBIR_NO_UNROLL_LOOP_START
2864+ do
2865+ {
2866+ int a,b;
2867+ STBIR_NO_UNROLL(sd);
2868+ a = ((int*)sd)[0];
2869+ b = ((int*)sd)[1];
2870+ ((int*)( sd + ofs_to_dest ))[0] = a;
2871+ ((int*)( sd + ofs_to_dest ))[1] = b;
2872+ sd += 8;
2873+ } while ( sd < s_end8 );
2874+ }
2875+
2876+ if ( sd == s_end )
2877+ return;
2878+ }
2879+
2880+ STBIR_NO_UNROLL_LOOP_START
2881+ do
2882+ {
2883+ STBIR_NO_UNROLL(sd);
2884+ *(int*)( sd + ofs_to_dest ) = *(int*) sd;
2885+ sd += 4;
2886+ } while ( sd < s_end );
2887+}
2888+
2889+#endif
2890+
2891+static float stbir__filter_trapezoid(float x, float scale, void * user_data)
2892+{
2893+ float halfscale = scale / 2;
2894+ float t = 0.5f + halfscale;
2895+ STBIR_ASSERT(scale <= 1);
2896+ STBIR__UNUSED(user_data);
2897+
2898+ if ( x < 0.0f ) x = -x;
2899+
2900+ if (x >= t)
2901+ return 0.0f;
2902+ else
2903+ {
2904+ float r = 0.5f - halfscale;
2905+ if (x <= r)
2906+ return 1.0f;
2907+ else
2908+ return (t - x) / scale;
2909+ }
2910+}
2911+
2912+static float stbir__support_trapezoid(float scale, void * user_data)
2913+{
2914+ STBIR__UNUSED(user_data);
2915+ return 0.5f + scale / 2.0f;
2916+}
2917+
2918+static float stbir__filter_triangle(float x, float s, void * user_data)
2919+{
2920+ STBIR__UNUSED(s);
2921+ STBIR__UNUSED(user_data);
2922+
2923+ if ( x < 0.0f ) x = -x;
2924+
2925+ if (x <= 1.0f)
2926+ return 1.0f - x;
2927+ else
2928+ return 0.0f;
2929+}
2930+
2931+static float stbir__filter_point(float x, float s, void * user_data)
2932+{
2933+ STBIR__UNUSED(x);
2934+ STBIR__UNUSED(s);
2935+ STBIR__UNUSED(user_data);
2936+
2937+ return 1.0f;
2938+}
2939+
2940+static float stbir__filter_cubic(float x, float s, void * user_data)
2941+{
2942+ STBIR__UNUSED(s);
2943+ STBIR__UNUSED(user_data);
2944+
2945+ if ( x < 0.0f ) x = -x;
2946+
2947+ if (x < 1.0f)
2948+ return (4.0f + x*x*(3.0f*x - 6.0f))/6.0f;
2949+ else if (x < 2.0f)
2950+ return (8.0f + x*(-12.0f + x*(6.0f - x)))/6.0f;
2951+
2952+ return (0.0f);
2953+}
2954+
2955+static float stbir__filter_catmullrom(float x, float s, void * user_data)
2956+{
2957+ STBIR__UNUSED(s);
2958+ STBIR__UNUSED(user_data);
2959+
2960+ if ( x < 0.0f ) x = -x;
2961+
2962+ if (x < 1.0f)
2963+ return 1.0f - x*x*(2.5f - 1.5f*x);
2964+ else if (x < 2.0f)
2965+ return 2.0f - x*(4.0f + x*(0.5f*x - 2.5f));
2966+
2967+ return (0.0f);
2968+}
2969+
2970+static float stbir__filter_mitchell(float x, float s, void * user_data)
2971+{
2972+ STBIR__UNUSED(s);
2973+ STBIR__UNUSED(user_data);
2974+
2975+ if ( x < 0.0f ) x = -x;
2976+
2977+ if (x < 1.0f)
2978+ return (16.0f + x*x*(21.0f * x - 36.0f))/18.0f;
2979+ else if (x < 2.0f)
2980+ return (32.0f + x*(-60.0f + x*(36.0f - 7.0f*x)))/18.0f;
2981+
2982+ return (0.0f);
2983+}
2984+
2985+static float stbir__support_zeropoint5(float s, void * user_data)
2986+{
2987+ STBIR__UNUSED(s);
2988+ STBIR__UNUSED(user_data);
2989+ return 0.5f;
2990+}
2991+
2992+static float stbir__support_one(float s, void * user_data)
2993+{
2994+ STBIR__UNUSED(s);
2995+ STBIR__UNUSED(user_data);
2996+ return 1;
2997+}
2998+
2999+static float stbir__support_two(float s, void * user_data)
3000+{
3001+ STBIR__UNUSED(s);
3002+ STBIR__UNUSED(user_data);
3003+ return 2;
3004+}
3005+
3006+// This is the maximum number of input samples that can affect an output sample
3007+// with the given filter from the output pixel's perspective
3008+static int stbir__get_filter_pixel_width(stbir__support_callback * support, float scale, void * user_data)
3009+{
3010+ STBIR_ASSERT(support != 0);
3011+
3012+ if ( scale >= ( 1.0f-stbir__small_float ) ) // upscale
3013+ return (int)STBIR_CEILF(support(1.0f/scale,user_data) * 2.0f);
3014+ else
3015+ return (int)STBIR_CEILF(support(scale,user_data) * 2.0f / scale);
3016+}
3017+
3018+// this is how many coefficents per run of the filter (which is different
3019+// from the filter_pixel_width depending on if we are scattering or gathering)
3020+static int stbir__get_coefficient_width(stbir__sampler * samp, int is_gather, void * user_data)
3021+{
3022+ float scale = samp->scale_info.scale;
3023+ stbir__support_callback * support = samp->filter_support;
3024+
3025+ switch( is_gather )
3026+ {
3027+ case 1:
3028+ return (int)STBIR_CEILF(support(1.0f / scale, user_data) * 2.0f);
3029+ case 2:
3030+ return (int)STBIR_CEILF(support(scale, user_data) * 2.0f / scale);
3031+ case 0:
3032+ return (int)STBIR_CEILF(support(scale, user_data) * 2.0f);
3033+ default:
3034+ STBIR_ASSERT( (is_gather >= 0 ) && (is_gather <= 2 ) );
3035+ return 0;
3036+ }
3037+}
3038+
3039+static int stbir__get_contributors(stbir__sampler * samp, int is_gather)
3040+{
3041+ if (is_gather)
3042+ return samp->scale_info.output_sub_size;
3043+ else
3044+ return (samp->scale_info.input_full_size + samp->filter_pixel_margin * 2);
3045+}
3046+
3047+static int stbir__edge_zero_full( int n, int max )
3048+{
3049+ STBIR__UNUSED(n);
3050+ STBIR__UNUSED(max);
3051+ return 0; // NOTREACHED
3052+}
3053+
3054+static int stbir__edge_clamp_full( int n, int max )
3055+{
3056+ if (n < 0)
3057+ return 0;
3058+
3059+ if (n >= max)
3060+ return max - 1;
3061+
3062+ return n; // NOTREACHED
3063+}
3064+
3065+static int stbir__edge_reflect_full( int n, int max )
3066+{
3067+ if (n < 0)
3068+ {
3069+ if (n > -max)
3070+ return -n;
3071+ else
3072+ return max - 1;
3073+ }
3074+
3075+ if (n >= max)
3076+ {
3077+ int max2 = max * 2;
3078+ if (n >= max2)
3079+ return 0;
3080+ else
3081+ return max2 - n - 1;
3082+ }
3083+
3084+ return n; // NOTREACHED
3085+}
3086+
3087+static int stbir__edge_wrap_full( int n, int max )
3088+{
3089+ if (n >= 0)
3090+ return (n % max);
3091+ else
3092+ {
3093+ int m = (-n) % max;
3094+
3095+ if (m != 0)
3096+ m = max - m;
3097+
3098+ return (m);
3099+ }
3100+}
3101+
3102+typedef int stbir__edge_wrap_func( int n, int max );
3103+static stbir__edge_wrap_func * stbir__edge_wrap_slow[] =
3104+{
3105+ stbir__edge_clamp_full, // STBIR_EDGE_CLAMP
3106+ stbir__edge_reflect_full, // STBIR_EDGE_REFLECT
3107+ stbir__edge_wrap_full, // STBIR_EDGE_WRAP
3108+ stbir__edge_zero_full, // STBIR_EDGE_ZERO
3109+};
3110+
3111+stbir__inline static int stbir__edge_wrap(stbir_edge edge, int n, int max)
3112+{
3113+ // avoid per-pixel switch
3114+ if (n >= 0 && n < max)
3115+ return n;
3116+ return stbir__edge_wrap_slow[edge]( n, max );
3117+}
3118+
3119+#define STBIR__MERGE_RUNS_PIXEL_THRESHOLD 16
3120+
3121+// get information on the extents of a sampler
3122+static void stbir__get_extents( stbir__sampler * samp, stbir__extents * scanline_extents )
3123+{
3124+ int j, stop;
3125+ int left_margin, right_margin;
3126+ int min_n = 0x7fffffff, max_n = -0x7fffffff;
3127+ int min_left = 0x7fffffff, max_left = -0x7fffffff;
3128+ int min_right = 0x7fffffff, max_right = -0x7fffffff;
3129+ stbir_edge edge = samp->edge;
3130+ stbir__contributors* contributors = samp->contributors;
3131+ int output_sub_size = samp->scale_info.output_sub_size;
3132+ int input_full_size = samp->scale_info.input_full_size;
3133+ int filter_pixel_margin = samp->filter_pixel_margin;
3134+
3135+ STBIR_ASSERT( samp->is_gather );
3136+
3137+ stop = output_sub_size;
3138+ for (j = 0; j < stop; j++ )
3139+ {
3140+ STBIR_ASSERT( contributors[j].n1 >= contributors[j].n0 );
3141+ if ( contributors[j].n0 < min_n )
3142+ {
3143+ min_n = contributors[j].n0;
3144+ stop = j + filter_pixel_margin; // if we find a new min, only scan another filter width
3145+ if ( stop > output_sub_size ) stop = output_sub_size;
3146+ }
3147+ }
3148+
3149+ stop = 0;
3150+ for (j = output_sub_size - 1; j >= stop; j-- )
3151+ {
3152+ STBIR_ASSERT( contributors[j].n1 >= contributors[j].n0 );
3153+ if ( contributors[j].n1 > max_n )
3154+ {
3155+ max_n = contributors[j].n1;
3156+ stop = j - filter_pixel_margin; // if we find a new max, only scan another filter width
3157+ if (stop<0) stop = 0;
3158+ }
3159+ }
3160+
3161+ STBIR_ASSERT( scanline_extents->conservative.n0 <= min_n );
3162+ STBIR_ASSERT( scanline_extents->conservative.n1 >= max_n );
3163+
3164+ // now calculate how much into the margins we really read
3165+ left_margin = 0;
3166+ if ( min_n < 0 )
3167+ {
3168+ left_margin = -min_n;
3169+ min_n = 0;
3170+ }
3171+
3172+ right_margin = 0;
3173+ if ( max_n >= input_full_size )
3174+ {
3175+ right_margin = max_n - input_full_size + 1;
3176+ max_n = input_full_size - 1;
3177+ }
3178+
3179+ // index 1 is margin pixel extents (how many pixels we hang over the edge)
3180+ scanline_extents->edge_sizes[0] = left_margin;
3181+ scanline_extents->edge_sizes[1] = right_margin;
3182+
3183+ // index 2 is pixels read from the input
3184+ scanline_extents->spans[0].n0 = min_n;
3185+ scanline_extents->spans[0].n1 = max_n;
3186+ scanline_extents->spans[0].pixel_offset_for_input = min_n;
3187+
3188+ // default to no other input range
3189+ scanline_extents->spans[1].n0 = 0;
3190+ scanline_extents->spans[1].n1 = -1;
3191+ scanline_extents->spans[1].pixel_offset_for_input = 0;
3192+
3193+ // don't have to do edge calc for zero clamp
3194+ if ( edge == STBIR_EDGE_ZERO )
3195+ return;
3196+
3197+ // convert margin pixels to the pixels within the input (min and max)
3198+ for( j = -left_margin ; j < 0 ; j++ )
3199+ {
3200+ int p = stbir__edge_wrap( edge, j, input_full_size );
3201+ if ( p < min_left )
3202+ min_left = p;
3203+ if ( p > max_left )
3204+ max_left = p;
3205+ }
3206+
3207+ for( j = input_full_size ; j < (input_full_size + right_margin) ; j++ )
3208+ {
3209+ int p = stbir__edge_wrap( edge, j, input_full_size );
3210+ if ( p < min_right )
3211+ min_right = p;
3212+ if ( p > max_right )
3213+ max_right = p;
3214+ }
3215+
3216+ // merge the left margin pixel region if it connects within 4 pixels of main pixel region
3217+ if ( min_left != 0x7fffffff )
3218+ {
3219+ if ( ( ( min_left <= min_n ) && ( ( max_left + STBIR__MERGE_RUNS_PIXEL_THRESHOLD ) >= min_n ) ) ||
3220+ ( ( min_n <= min_left ) && ( ( max_n + STBIR__MERGE_RUNS_PIXEL_THRESHOLD ) >= max_left ) ) )
3221+ {
3222+ scanline_extents->spans[0].n0 = min_n = stbir__min( min_n, min_left );
3223+ scanline_extents->spans[0].n1 = max_n = stbir__max( max_n, max_left );
3224+ scanline_extents->spans[0].pixel_offset_for_input = min_n;
3225+ left_margin = 0;
3226+ }
3227+ }
3228+
3229+ // merge the right margin pixel region if it connects within 4 pixels of main pixel region
3230+ if ( min_right != 0x7fffffff )
3231+ {
3232+ if ( ( ( min_right <= min_n ) && ( ( max_right + STBIR__MERGE_RUNS_PIXEL_THRESHOLD ) >= min_n ) ) ||
3233+ ( ( min_n <= min_right ) && ( ( max_n + STBIR__MERGE_RUNS_PIXEL_THRESHOLD ) >= max_right ) ) )
3234+ {
3235+ scanline_extents->spans[0].n0 = min_n = stbir__min( min_n, min_right );
3236+ scanline_extents->spans[0].n1 = max_n = stbir__max( max_n, max_right );
3237+ scanline_extents->spans[0].pixel_offset_for_input = min_n;
3238+ right_margin = 0;
3239+ }
3240+ }
3241+
3242+ STBIR_ASSERT( scanline_extents->conservative.n0 <= min_n );
3243+ STBIR_ASSERT( scanline_extents->conservative.n1 >= max_n );
3244+
3245+ // you get two ranges when you have the WRAP edge mode and you are doing just the a piece of the resize
3246+ // so you need to get a second run of pixels from the opposite side of the scanline (which you
3247+ // wouldn't need except for WRAP)
3248+
3249+
3250+ // if we can't merge the min_left range, add it as a second range
3251+ if ( ( left_margin ) && ( min_left != 0x7fffffff ) )
3252+ {
3253+ stbir__span * newspan = scanline_extents->spans + 1;
3254+ STBIR_ASSERT( right_margin == 0 );
3255+ if ( min_left < scanline_extents->spans[0].n0 )
3256+ {
3257+ scanline_extents->spans[1].pixel_offset_for_input = scanline_extents->spans[0].n0;
3258+ scanline_extents->spans[1].n0 = scanline_extents->spans[0].n0;
3259+ scanline_extents->spans[1].n1 = scanline_extents->spans[0].n1;
3260+ --newspan;
3261+ }
3262+ newspan->pixel_offset_for_input = min_left;
3263+ newspan->n0 = -left_margin;
3264+ newspan->n1 = ( max_left - min_left ) - left_margin;
3265+ scanline_extents->edge_sizes[0] = 0; // don't need to copy the left margin, since we are directly decoding into the margin
3266+ }
3267+ // if we can't merge the min_right range, add it as a second range
3268+ else
3269+ if ( ( right_margin ) && ( min_right != 0x7fffffff ) )
3270+ {
3271+ stbir__span * newspan = scanline_extents->spans + 1;
3272+ if ( min_right < scanline_extents->spans[0].n0 )
3273+ {
3274+ scanline_extents->spans[1].pixel_offset_for_input = scanline_extents->spans[0].n0;
3275+ scanline_extents->spans[1].n0 = scanline_extents->spans[0].n0;
3276+ scanline_extents->spans[1].n1 = scanline_extents->spans[0].n1;
3277+ --newspan;
3278+ }
3279+ newspan->pixel_offset_for_input = min_right;
3280+ newspan->n0 = scanline_extents->spans[1].n1 + 1;
3281+ newspan->n1 = scanline_extents->spans[1].n1 + 1 + ( max_right - min_right );
3282+ scanline_extents->edge_sizes[1] = 0; // don't need to copy the right margin, since we are directly decoding into the margin
3283+ }
3284+
3285+ // sort the spans into write output order
3286+ if ( ( scanline_extents->spans[1].n1 > scanline_extents->spans[1].n0 ) && ( scanline_extents->spans[0].n0 > scanline_extents->spans[1].n0 ) )
3287+ {
3288+ stbir__span tspan = scanline_extents->spans[0];
3289+ scanline_extents->spans[0] = scanline_extents->spans[1];
3290+ scanline_extents->spans[1] = tspan;
3291+ }
3292+}
3293+
3294+static void stbir__calculate_in_pixel_range( int * first_pixel, int * last_pixel, float out_pixel_center, float out_filter_radius, float inv_scale, float out_shift, int input_size, stbir_edge edge )
3295+{
3296+ int first, last;
3297+ float out_pixel_influence_lowerbound = out_pixel_center - out_filter_radius;
3298+ float out_pixel_influence_upperbound = out_pixel_center + out_filter_radius;
3299+
3300+ float in_pixel_influence_lowerbound = (out_pixel_influence_lowerbound + out_shift) * inv_scale;
3301+ float in_pixel_influence_upperbound = (out_pixel_influence_upperbound + out_shift) * inv_scale;
3302+
3303+ first = (int)(STBIR_FLOORF(in_pixel_influence_lowerbound + 0.5f));
3304+ last = (int)(STBIR_FLOORF(in_pixel_influence_upperbound - 0.5f));
3305+ if ( last < first ) last = first; // point sample mode can span a value *right* at 0.5, and cause these to cross
3306+
3307+ if ( edge == STBIR_EDGE_WRAP )
3308+ {
3309+ if ( first < -input_size )
3310+ first = -input_size;
3311+ if ( last >= (input_size*2))
3312+ last = (input_size*2) - 1;
3313+ }
3314+
3315+ *first_pixel = first;
3316+ *last_pixel = last;
3317+}
3318+
3319+static void stbir__calculate_coefficients_for_gather_upsample( float out_filter_radius, stbir__kernel_callback * kernel, stbir__scale_info * scale_info, int num_contributors, stbir__contributors* contributors, float* coefficient_group, int coefficient_width, stbir_edge edge, void * user_data )
3320+{
3321+ int n, end;
3322+ float inv_scale = scale_info->inv_scale;
3323+ float out_shift = scale_info->pixel_shift;
3324+ int input_size = scale_info->input_full_size;
3325+ int numerator = scale_info->scale_numerator;
3326+ int polyphase = ( ( scale_info->scale_is_rational ) && ( numerator < num_contributors ) );
3327+
3328+ // Looping through out pixels
3329+ end = num_contributors; if ( polyphase ) end = numerator;
3330+ for (n = 0; n < end; n++)
3331+ {
3332+ int i;
3333+ int last_non_zero;
3334+ float out_pixel_center = (float)n + 0.5f;
3335+ float in_center_of_out = (out_pixel_center + out_shift) * inv_scale;
3336+
3337+ int in_first_pixel, in_last_pixel;
3338+
3339+ stbir__calculate_in_pixel_range( &in_first_pixel, &in_last_pixel, out_pixel_center, out_filter_radius, inv_scale, out_shift, input_size, edge );
3340+
3341+ // make sure we never generate a range larger than our precalculated coeff width
3342+ // this only happens in point sample mode, but it's a good safe thing to do anyway
3343+ if ( ( in_last_pixel - in_first_pixel + 1 ) > coefficient_width )
3344+ in_last_pixel = in_first_pixel + coefficient_width - 1;
3345+
3346+ last_non_zero = -1;
3347+ for (i = 0; i <= in_last_pixel - in_first_pixel; i++)
3348+ {
3349+ float in_pixel_center = (float)(i + in_first_pixel) + 0.5f;
3350+ float coeff = kernel(in_center_of_out - in_pixel_center, inv_scale, user_data);
3351+
3352+ // kill denormals
3353+ if ( ( ( coeff < stbir__small_float ) && ( coeff > -stbir__small_float ) ) )
3354+ {
3355+ if ( i == 0 ) // if we're at the front, just eat zero contributors
3356+ {
3357+ STBIR_ASSERT ( ( in_last_pixel - in_first_pixel ) != 0 ); // there should be at least one contrib
3358+ ++in_first_pixel;
3359+ i--;
3360+ continue;
3361+ }
3362+ coeff = 0; // make sure is fully zero (should keep denormals away)
3363+ }
3364+ else
3365+ last_non_zero = i;
3366+
3367+ coefficient_group[i] = coeff;
3368+ }
3369+
3370+ in_last_pixel = last_non_zero+in_first_pixel; // kills trailing zeros
3371+ contributors->n0 = in_first_pixel;
3372+ contributors->n1 = in_last_pixel;
3373+
3374+ STBIR_ASSERT(contributors->n1 >= contributors->n0);
3375+
3376+ ++contributors;
3377+ coefficient_group += coefficient_width;
3378+ }
3379+}
3380+
3381+static void stbir__insert_coeff( stbir__contributors * contribs, float * coeffs, int new_pixel, float new_coeff, int max_width )
3382+{
3383+ if ( contribs->n1 < contribs->n0 ) // this first clause should never happen, but handle in case
3384+ {
3385+ contribs->n0 = contribs->n1 = new_pixel;
3386+ coeffs[0] = new_coeff;
3387+ }
3388+ else if ( new_pixel <= contribs->n1 ) // before the end
3389+ {
3390+ if ( new_pixel < contribs->n0 ) // before the front?
3391+ {
3392+ if ( ( contribs->n1 - new_pixel + 1 ) <= max_width )
3393+ {
3394+ int j, o = contribs->n0 - new_pixel;
3395+ for ( j = contribs->n1 - contribs->n0 ; j >= 0 ; j-- )
3396+ coeffs[ j + o ] = coeffs[ j ];
3397+ for ( j = 1 ; j < o ; j++ )
3398+ coeffs[ j ] = 0;
3399+ coeffs[ 0 ] = new_coeff;
3400+ contribs->n0 = new_pixel;
3401+ }
3402+ }
3403+ else
3404+ {
3405+ // add new weight to existing coeff if already there
3406+ coeffs[ new_pixel - contribs->n0 ] += new_coeff;
3407+ }
3408+ }
3409+ else
3410+ {
3411+ if ( ( new_pixel - contribs->n0 + 1 ) <= max_width )
3412+ {
3413+ int j, e = new_pixel - contribs->n0;
3414+ for( j = ( contribs->n1 - contribs->n0 ) + 1 ; j < e ; j++ ) // clear in-betweens coeffs if there are any
3415+ coeffs[j] = 0;
3416+
3417+ coeffs[ e ] = new_coeff;
3418+ contribs->n1 = new_pixel;
3419+ }
3420+ }
3421+}
3422+
3423+static void stbir__calculate_out_pixel_range( int * first_pixel, int * last_pixel, float in_pixel_center, float in_pixels_radius, float scale, float out_shift, int out_size )
3424+{
3425+ float in_pixel_influence_lowerbound = in_pixel_center - in_pixels_radius;
3426+ float in_pixel_influence_upperbound = in_pixel_center + in_pixels_radius;
3427+ float out_pixel_influence_lowerbound = in_pixel_influence_lowerbound * scale - out_shift;
3428+ float out_pixel_influence_upperbound = in_pixel_influence_upperbound * scale - out_shift;
3429+ int out_first_pixel = (int)(STBIR_FLOORF(out_pixel_influence_lowerbound + 0.5f));
3430+ int out_last_pixel = (int)(STBIR_FLOORF(out_pixel_influence_upperbound - 0.5f));
3431+
3432+ if ( out_first_pixel < 0 )
3433+ out_first_pixel = 0;
3434+ if ( out_last_pixel >= out_size )
3435+ out_last_pixel = out_size - 1;
3436+ *first_pixel = out_first_pixel;
3437+ *last_pixel = out_last_pixel;
3438+}
3439+
3440+static void stbir__calculate_coefficients_for_gather_downsample( int start, int end, float in_pixels_radius, stbir__kernel_callback * kernel, stbir__scale_info * scale_info, int coefficient_width, int num_contributors, stbir__contributors * contributors, float * coefficient_group, void * user_data )
3441+{
3442+ int in_pixel;
3443+ int i;
3444+ int first_out_inited = -1;
3445+ float scale = scale_info->scale;
3446+ float out_shift = scale_info->pixel_shift;
3447+ int out_size = scale_info->output_sub_size;
3448+ int numerator = scale_info->scale_numerator;
3449+ int polyphase = ( ( scale_info->scale_is_rational ) && ( numerator < out_size ) );
3450+
3451+ STBIR__UNUSED(num_contributors);
3452+
3453+ // Loop through the input pixels
3454+ for (in_pixel = start; in_pixel < end; in_pixel++)
3455+ {
3456+ float in_pixel_center = (float)in_pixel + 0.5f;
3457+ float out_center_of_in = in_pixel_center * scale - out_shift;
3458+ int out_first_pixel, out_last_pixel;
3459+
3460+ stbir__calculate_out_pixel_range( &out_first_pixel, &out_last_pixel, in_pixel_center, in_pixels_radius, scale, out_shift, out_size );
3461+
3462+ if ( out_first_pixel > out_last_pixel )
3463+ continue;
3464+
3465+ // clamp or exit if we are using polyphase filtering, and the limit is up
3466+ if ( polyphase )
3467+ {
3468+ // when polyphase, you only have to do coeffs up to the numerator count
3469+ if ( out_first_pixel == numerator )
3470+ break;
3471+
3472+ // don't do any extra work, clamp last pixel at numerator too
3473+ if ( out_last_pixel >= numerator )
3474+ out_last_pixel = numerator - 1;
3475+ }
3476+
3477+ for (i = 0; i <= out_last_pixel - out_first_pixel; i++)
3478+ {
3479+ float out_pixel_center = (float)(i + out_first_pixel) + 0.5f;
3480+ float x = out_pixel_center - out_center_of_in;
3481+ float coeff = kernel(x, scale, user_data) * scale;
3482+
3483+ // kill the coeff if it's too small (avoid denormals)
3484+ if ( ( ( coeff < stbir__small_float ) && ( coeff > -stbir__small_float ) ) )
3485+ coeff = 0.0f;
3486+
3487+ {
3488+ int out = i + out_first_pixel;
3489+ float * coeffs = coefficient_group + out * coefficient_width;
3490+ stbir__contributors * contribs = contributors + out;
3491+
3492+ // is this the first time this output pixel has been seen? Init it.
3493+ if ( out > first_out_inited )
3494+ {
3495+ STBIR_ASSERT( out == ( first_out_inited + 1 ) ); // ensure we have only advanced one at time
3496+ first_out_inited = out;
3497+ contribs->n0 = in_pixel;
3498+ contribs->n1 = in_pixel;
3499+ coeffs[0] = coeff;
3500+ }
3501+ else
3502+ {
3503+ // insert on end (always in order)
3504+ if ( coeffs[0] == 0.0f ) // if the first coefficent is zero, then zap it for this coeffs
3505+ {
3506+ STBIR_ASSERT( ( in_pixel - contribs->n0 ) == 1 ); // ensure that when we zap, we're at the 2nd pos
3507+ contribs->n0 = in_pixel;
3508+ }
3509+ contribs->n1 = in_pixel;
3510+ STBIR_ASSERT( ( in_pixel - contribs->n0 ) < coefficient_width );
3511+ coeffs[in_pixel - contribs->n0] = coeff;
3512+ }
3513+ }
3514+ }
3515+ }
3516+}
3517+
3518+#ifdef STBIR_RENORMALIZE_IN_FLOAT
3519+#define STBIR_RENORM_TYPE float
3520+#else
3521+#define STBIR_RENORM_TYPE double
3522+#endif
3523+
3524+static void stbir__cleanup_gathered_coefficients( stbir_edge edge, stbir__filter_extent_info* filter_info, stbir__scale_info * scale_info, int num_contributors, stbir__contributors* contributors, float * coefficient_group, int coefficient_width )
3525+{
3526+ int input_size = scale_info->input_full_size;
3527+ int input_last_n1 = input_size - 1;
3528+ int n, end;
3529+ int lowest = 0x7fffffff;
3530+ int highest = -0x7fffffff;
3531+ int widest = -1;
3532+ int numerator = scale_info->scale_numerator;
3533+ int denominator = scale_info->scale_denominator;
3534+ int polyphase = ( ( scale_info->scale_is_rational ) && ( numerator < num_contributors ) );
3535+ float * coeffs;
3536+ stbir__contributors * contribs;
3537+
3538+ // weight all the coeffs for each sample
3539+ coeffs = coefficient_group;
3540+ contribs = contributors;
3541+ end = num_contributors; if ( polyphase ) end = numerator;
3542+ for (n = 0; n < end; n++)
3543+ {
3544+ int i;
3545+ STBIR_RENORM_TYPE filter_scale, total_filter = 0;
3546+ int e;
3547+
3548+ // add all contribs
3549+ e = contribs->n1 - contribs->n0;
3550+ for( i = 0 ; i <= e ; i++ )
3551+ {
3552+ total_filter += (STBIR_RENORM_TYPE) coeffs[i];
3553+ STBIR_ASSERT( ( coeffs[i] >= -2.0f ) && ( coeffs[i] <= 2.0f ) ); // check for wonky weights
3554+ }
3555+
3556+ // rescale
3557+ if ( ( total_filter < stbir__small_float ) && ( total_filter > -stbir__small_float ) )
3558+ {
3559+ // all coeffs are extremely small, just zero it
3560+ contribs->n1 = contribs->n0;
3561+ coeffs[0] = 0.0f;
3562+ }
3563+ else
3564+ {
3565+ // if the total isn't 1.0, rescale everything
3566+ if ( ( total_filter < (1.0f-stbir__small_float) ) || ( total_filter > (1.0f+stbir__small_float) ) )
3567+ {
3568+ filter_scale = ((STBIR_RENORM_TYPE)1.0) / total_filter;
3569+
3570+ // scale them all
3571+ for (i = 0; i <= e; i++)
3572+ coeffs[i] = (float) ( coeffs[i] * filter_scale );
3573+ }
3574+ }
3575+ ++contribs;
3576+ coeffs += coefficient_width;
3577+ }
3578+
3579+ // if we have a rational for the scale, we can exploit the polyphaseness to not calculate
3580+ // most of the coefficients, so we copy them here
3581+ if ( polyphase )
3582+ {
3583+ stbir__contributors * prev_contribs = contributors;
3584+ stbir__contributors * cur_contribs = contributors + numerator;
3585+
3586+ for( n = numerator ; n < num_contributors ; n++ )
3587+ {
3588+ cur_contribs->n0 = prev_contribs->n0 + denominator;
3589+ cur_contribs->n1 = prev_contribs->n1 + denominator;
3590+ ++cur_contribs;
3591+ ++prev_contribs;
3592+ }
3593+ stbir_overlapping_memcpy( coefficient_group + numerator * coefficient_width, coefficient_group, ( num_contributors - numerator ) * coefficient_width * sizeof( coeffs[ 0 ] ) );
3594+ }
3595+
3596+ coeffs = coefficient_group;
3597+ contribs = contributors;
3598+
3599+ for (n = 0; n < num_contributors; n++)
3600+ {
3601+ int i;
3602+
3603+ // in zero edge mode, just remove out of bounds contribs completely (since their weights are accounted for now)
3604+ if ( edge == STBIR_EDGE_ZERO )
3605+ {
3606+ // shrink the right side if necessary
3607+ if ( contribs->n1 > input_last_n1 )
3608+ contribs->n1 = input_last_n1;
3609+
3610+ // shrink the left side
3611+ if ( contribs->n0 < 0 )
3612+ {
3613+ int j, left, skips = 0;
3614+
3615+ skips = -contribs->n0;
3616+ contribs->n0 = 0;
3617+
3618+ // now move down the weights
3619+ left = contribs->n1 - contribs->n0 + 1;
3620+ if ( left > 0 )
3621+ {
3622+ for( j = 0 ; j < left ; j++ )
3623+ coeffs[ j ] = coeffs[ j + skips ];
3624+ }
3625+ }
3626+ }
3627+ else if ( ( edge == STBIR_EDGE_CLAMP ) || ( edge == STBIR_EDGE_REFLECT ) )
3628+ {
3629+ // for clamp and reflect, calculate the true inbounds position (based on edge type) and just add that to the existing weight
3630+
3631+ // right hand side first
3632+ if ( contribs->n1 > input_last_n1 )
3633+ {
3634+ int start = contribs->n0;
3635+ int endi = contribs->n1;
3636+ contribs->n1 = input_last_n1;
3637+ for( i = input_size; i <= endi; i++ )
3638+ stbir__insert_coeff( contribs, coeffs, stbir__edge_wrap_slow[edge]( i, input_size ), coeffs[i-start], coefficient_width );
3639+ }
3640+
3641+ // now check left hand edge
3642+ if ( contribs->n0 < 0 )
3643+ {
3644+ int save_n0;
3645+ float save_n0_coeff;
3646+ float * c = coeffs - ( contribs->n0 + 1 );
3647+
3648+ // reinsert the coeffs with it reflected or clamped (insert accumulates, if the coeffs exist)
3649+ for( i = -1 ; i > contribs->n0 ; i-- )
3650+ stbir__insert_coeff( contribs, coeffs, stbir__edge_wrap_slow[edge]( i, input_size ), *c--, coefficient_width );
3651+ save_n0 = contribs->n0;
3652+ save_n0_coeff = c[0]; // save it, since we didn't do the final one (i==n0), because there might be too many coeffs to hold (before we resize)!
3653+
3654+ // now slide all the coeffs down (since we have accumulated them in the positive contribs) and reset the first contrib
3655+ contribs->n0 = 0;
3656+ for(i = 0 ; i <= contribs->n1 ; i++ )
3657+ coeffs[i] = coeffs[i-save_n0];
3658+
3659+ // now that we have shrunk down the contribs, we insert the first one safely
3660+ stbir__insert_coeff( contribs, coeffs, stbir__edge_wrap_slow[edge]( save_n0, input_size ), save_n0_coeff, coefficient_width );
3661+ }
3662+ }
3663+
3664+ if ( contribs->n0 <= contribs->n1 )
3665+ {
3666+ int diff = contribs->n1 - contribs->n0 + 1;
3667+ while ( diff && ( coeffs[ diff-1 ] == 0.0f ) )
3668+ --diff;
3669+
3670+ contribs->n1 = contribs->n0 + diff - 1;
3671+
3672+ if ( contribs->n0 <= contribs->n1 )
3673+ {
3674+ if ( contribs->n0 < lowest )
3675+ lowest = contribs->n0;
3676+ if ( contribs->n1 > highest )
3677+ highest = contribs->n1;
3678+ if ( diff > widest )
3679+ widest = diff;
3680+ }
3681+
3682+ // re-zero out unused coefficients (if any)
3683+ for( i = diff ; i < coefficient_width ; i++ )
3684+ coeffs[i] = 0.0f;
3685+ }
3686+
3687+ ++contribs;
3688+ coeffs += coefficient_width;
3689+ }
3690+ filter_info->lowest = lowest;
3691+ filter_info->highest = highest;
3692+ filter_info->widest = widest;
3693+}
3694+
3695+#undef STBIR_RENORM_TYPE
3696+
3697+static int stbir__pack_coefficients( int num_contributors, stbir__contributors* contributors, float * coefficents, int coefficient_width, int widest, int row0, int row1 )
3698+{
3699+ #define STBIR_MOVE_1( dest, src ) { STBIR_NO_UNROLL(dest); ((stbir_uint32*)(dest))[0] = ((stbir_uint32*)(src))[0]; }
3700+ #define STBIR_MOVE_2( dest, src ) { STBIR_NO_UNROLL(dest); ((stbir_uint64*)(dest))[0] = ((stbir_uint64*)(src))[0]; }
3701+ #ifdef STBIR_SIMD
3702+ #define STBIR_MOVE_4( dest, src ) { stbir__simdf t; STBIR_NO_UNROLL(dest); stbir__simdf_load( t, src ); stbir__simdf_store( dest, t ); }
3703+ #else
3704+ #define STBIR_MOVE_4( dest, src ) { STBIR_NO_UNROLL(dest); ((stbir_uint64*)(dest))[0] = ((stbir_uint64*)(src))[0]; ((stbir_uint64*)(dest))[1] = ((stbir_uint64*)(src))[1]; }
3705+ #endif
3706+
3707+ int row_end = row1 + 1;
3708+ STBIR__UNUSED( row0 ); // only used in an assert
3709+
3710+ if ( coefficient_width != widest )
3711+ {
3712+ float * pc = coefficents;
3713+ float * coeffs = coefficents;
3714+ float * pc_end = coefficents + num_contributors * widest;
3715+ switch( widest )
3716+ {
3717+ case 1:
3718+ STBIR_NO_UNROLL_LOOP_START
3719+ do {
3720+ STBIR_MOVE_1( pc, coeffs );
3721+ ++pc;
3722+ coeffs += coefficient_width;
3723+ } while ( pc < pc_end );
3724+ break;
3725+ case 2:
3726+ STBIR_NO_UNROLL_LOOP_START
3727+ do {
3728+ STBIR_MOVE_2( pc, coeffs );
3729+ pc += 2;
3730+ coeffs += coefficient_width;
3731+ } while ( pc < pc_end );
3732+ break;
3733+ case 3:
3734+ STBIR_NO_UNROLL_LOOP_START
3735+ do {
3736+ STBIR_MOVE_2( pc, coeffs );
3737+ STBIR_MOVE_1( pc+2, coeffs+2 );
3738+ pc += 3;
3739+ coeffs += coefficient_width;
3740+ } while ( pc < pc_end );
3741+ break;
3742+ case 4:
3743+ STBIR_NO_UNROLL_LOOP_START
3744+ do {
3745+ STBIR_MOVE_4( pc, coeffs );
3746+ pc += 4;
3747+ coeffs += coefficient_width;
3748+ } while ( pc < pc_end );
3749+ break;
3750+ case 5:
3751+ STBIR_NO_UNROLL_LOOP_START
3752+ do {
3753+ STBIR_MOVE_4( pc, coeffs );
3754+ STBIR_MOVE_1( pc+4, coeffs+4 );
3755+ pc += 5;
3756+ coeffs += coefficient_width;
3757+ } while ( pc < pc_end );
3758+ break;
3759+ case 6:
3760+ STBIR_NO_UNROLL_LOOP_START
3761+ do {
3762+ STBIR_MOVE_4( pc, coeffs );
3763+ STBIR_MOVE_2( pc+4, coeffs+4 );
3764+ pc += 6;
3765+ coeffs += coefficient_width;
3766+ } while ( pc < pc_end );
3767+ break;
3768+ case 7:
3769+ STBIR_NO_UNROLL_LOOP_START
3770+ do {
3771+ STBIR_MOVE_4( pc, coeffs );
3772+ STBIR_MOVE_2( pc+4, coeffs+4 );
3773+ STBIR_MOVE_1( pc+6, coeffs+6 );
3774+ pc += 7;
3775+ coeffs += coefficient_width;
3776+ } while ( pc < pc_end );
3777+ break;
3778+ case 8:
3779+ STBIR_NO_UNROLL_LOOP_START
3780+ do {
3781+ STBIR_MOVE_4( pc, coeffs );
3782+ STBIR_MOVE_4( pc+4, coeffs+4 );
3783+ pc += 8;
3784+ coeffs += coefficient_width;
3785+ } while ( pc < pc_end );
3786+ break;
3787+ case 9:
3788+ STBIR_NO_UNROLL_LOOP_START
3789+ do {
3790+ STBIR_MOVE_4( pc, coeffs );
3791+ STBIR_MOVE_4( pc+4, coeffs+4 );
3792+ STBIR_MOVE_1( pc+8, coeffs+8 );
3793+ pc += 9;
3794+ coeffs += coefficient_width;
3795+ } while ( pc < pc_end );
3796+ break;
3797+ case 10:
3798+ STBIR_NO_UNROLL_LOOP_START
3799+ do {
3800+ STBIR_MOVE_4( pc, coeffs );
3801+ STBIR_MOVE_4( pc+4, coeffs+4 );
3802+ STBIR_MOVE_2( pc+8, coeffs+8 );
3803+ pc += 10;
3804+ coeffs += coefficient_width;
3805+ } while ( pc < pc_end );
3806+ break;
3807+ case 11:
3808+ STBIR_NO_UNROLL_LOOP_START
3809+ do {
3810+ STBIR_MOVE_4( pc, coeffs );
3811+ STBIR_MOVE_4( pc+4, coeffs+4 );
3812+ STBIR_MOVE_2( pc+8, coeffs+8 );
3813+ STBIR_MOVE_1( pc+10, coeffs+10 );
3814+ pc += 11;
3815+ coeffs += coefficient_width;
3816+ } while ( pc < pc_end );
3817+ break;
3818+ case 12:
3819+ STBIR_NO_UNROLL_LOOP_START
3820+ do {
3821+ STBIR_MOVE_4( pc, coeffs );
3822+ STBIR_MOVE_4( pc+4, coeffs+4 );
3823+ STBIR_MOVE_4( pc+8, coeffs+8 );
3824+ pc += 12;
3825+ coeffs += coefficient_width;
3826+ } while ( pc < pc_end );
3827+ break;
3828+ default:
3829+ STBIR_NO_UNROLL_LOOP_START
3830+ do {
3831+ float * copy_end = pc + widest - 4;
3832+ float * c = coeffs;
3833+ do {
3834+ STBIR_NO_UNROLL( pc );
3835+ STBIR_MOVE_4( pc, c );
3836+ pc += 4;
3837+ c += 4;
3838+ } while ( pc <= copy_end );
3839+ copy_end += 4;
3840+ STBIR_NO_UNROLL_LOOP_START
3841+ while ( pc < copy_end )
3842+ {
3843+ STBIR_MOVE_1( pc, c );
3844+ ++pc; ++c;
3845+ }
3846+ coeffs += coefficient_width;
3847+ } while ( pc < pc_end );
3848+ break;
3849+ }
3850+ }
3851+
3852+ // some horizontal routines read one float off the end (which is then masked off), so put in a sentinel so we don't read an snan or denormal
3853+ coefficents[ widest * num_contributors ] = 8888.0f;
3854+
3855+ // the minimum we might read for unrolled filters widths is 12. So, we need to
3856+ // make sure we never read outside the decode buffer, by possibly moving
3857+ // the sample area back into the scanline, and putting zeros weights first.
3858+ // we start on the right edge and check until we're well past the possible
3859+ // clip area (2*widest).
3860+ {
3861+ stbir__contributors * contribs = contributors + num_contributors - 1;
3862+ float * coeffs = coefficents + widest * ( num_contributors - 1 );
3863+
3864+ // go until no chance of clipping (this is usually less than 8 lops)
3865+ while ( ( contribs >= contributors ) && ( ( contribs->n0 + widest*2 ) >= row_end ) )
3866+ {
3867+ // might we clip??
3868+ if ( ( contribs->n0 + widest ) > row_end )
3869+ {
3870+ int stop_range = widest;
3871+
3872+ // if range is larger than 12, it will be handled by generic loops that can terminate on the exact length
3873+ // of this contrib n1, instead of a fixed widest amount - so calculate this
3874+ if ( widest > 12 )
3875+ {
3876+ int mod;
3877+
3878+ // how far will be read in the n_coeff loop (which depends on the widest count mod4);
3879+ mod = widest & 3;
3880+ stop_range = ( ( ( contribs->n1 - contribs->n0 + 1 ) - mod + 3 ) & ~3 ) + mod;
3881+
3882+ // the n_coeff loops do a minimum amount of coeffs, so factor that in!
3883+ if ( stop_range < ( 8 + mod ) ) stop_range = 8 + mod;
3884+ }
3885+
3886+ // now see if we still clip with the refined range
3887+ if ( ( contribs->n0 + stop_range ) > row_end )
3888+ {
3889+ int new_n0 = row_end - stop_range;
3890+ int num = contribs->n1 - contribs->n0 + 1;
3891+ int backup = contribs->n0 - new_n0;
3892+ float * from_co = coeffs + num - 1;
3893+ float * to_co = from_co + backup;
3894+
3895+ STBIR_ASSERT( ( new_n0 >= row0 ) && ( new_n0 < contribs->n0 ) );
3896+
3897+ // move the coeffs over
3898+ while( num )
3899+ {
3900+ *to_co-- = *from_co--;
3901+ --num;
3902+ }
3903+ // zero new positions
3904+ while ( to_co >= coeffs )
3905+ *to_co-- = 0;
3906+ // set new start point
3907+ contribs->n0 = new_n0;
3908+ if ( widest > 12 )
3909+ {
3910+ int mod;
3911+
3912+ // how far will be read in the n_coeff loop (which depends on the widest count mod4);
3913+ mod = widest & 3;
3914+ stop_range = ( ( ( contribs->n1 - contribs->n0 + 1 ) - mod + 3 ) & ~3 ) + mod;
3915+
3916+ // the n_coeff loops do a minimum amount of coeffs, so factor that in!
3917+ if ( stop_range < ( 8 + mod ) ) stop_range = 8 + mod;
3918+ }
3919+ }
3920+ }
3921+ --contribs;
3922+ coeffs -= widest;
3923+ }
3924+ }
3925+
3926+ return widest;
3927+ #undef STBIR_MOVE_1
3928+ #undef STBIR_MOVE_2
3929+ #undef STBIR_MOVE_4
3930+}
3931+
3932+static void stbir__calculate_filters( stbir__sampler * samp, stbir__sampler * other_axis_for_pivot, void * user_data STBIR_ONLY_PROFILE_BUILD_GET_INFO )
3933+{
3934+ int n;
3935+ float scale = samp->scale_info.scale;
3936+ stbir__kernel_callback * kernel = samp->filter_kernel;
3937+ stbir__support_callback * support = samp->filter_support;
3938+ float inv_scale = samp->scale_info.inv_scale;
3939+ int input_full_size = samp->scale_info.input_full_size;
3940+ int gather_num_contributors = samp->num_contributors;
3941+ stbir__contributors* gather_contributors = samp->contributors;
3942+ float * gather_coeffs = samp->coefficients;
3943+ int gather_coefficient_width = samp->coefficient_width;
3944+
3945+ switch ( samp->is_gather )
3946+ {
3947+ case 1: // gather upsample
3948+ {
3949+ float out_pixels_radius = support(inv_scale,user_data) * scale;
3950+
3951+ stbir__calculate_coefficients_for_gather_upsample( out_pixels_radius, kernel, &samp->scale_info, gather_num_contributors, gather_contributors, gather_coeffs, gather_coefficient_width, samp->edge, user_data );
3952+
3953+ STBIR_PROFILE_BUILD_START( cleanup );
3954+ stbir__cleanup_gathered_coefficients( samp->edge, &samp->extent_info, &samp->scale_info, gather_num_contributors, gather_contributors, gather_coeffs, gather_coefficient_width );
3955+ STBIR_PROFILE_BUILD_END( cleanup );
3956+ }
3957+ break;
3958+
3959+ case 0: // scatter downsample (only on vertical)
3960+ case 2: // gather downsample
3961+ {
3962+ float in_pixels_radius = support(scale,user_data) * inv_scale;
3963+ int filter_pixel_margin = samp->filter_pixel_margin;
3964+ int input_end = input_full_size + filter_pixel_margin;
3965+
3966+ // if this is a scatter, we do a downsample gather to get the coeffs, and then pivot after
3967+ if ( !samp->is_gather )
3968+ {
3969+ // check if we are using the same gather downsample on the horizontal as this vertical,
3970+ // if so, then we don't have to generate them, we can just pivot from the horizontal.
3971+ if ( other_axis_for_pivot )
3972+ {
3973+ gather_contributors = other_axis_for_pivot->contributors;
3974+ gather_coeffs = other_axis_for_pivot->coefficients;
3975+ gather_coefficient_width = other_axis_for_pivot->coefficient_width;
3976+ gather_num_contributors = other_axis_for_pivot->num_contributors;
3977+ samp->extent_info.lowest = other_axis_for_pivot->extent_info.lowest;
3978+ samp->extent_info.highest = other_axis_for_pivot->extent_info.highest;
3979+ samp->extent_info.widest = other_axis_for_pivot->extent_info.widest;
3980+ goto jump_right_to_pivot;
3981+ }
3982+
3983+ gather_contributors = samp->gather_prescatter_contributors;
3984+ gather_coeffs = samp->gather_prescatter_coefficients;
3985+ gather_coefficient_width = samp->gather_prescatter_coefficient_width;
3986+ gather_num_contributors = samp->gather_prescatter_num_contributors;
3987+ }
3988+
3989+ stbir__calculate_coefficients_for_gather_downsample( -filter_pixel_margin, input_end, in_pixels_radius, kernel, &samp->scale_info, gather_coefficient_width, gather_num_contributors, gather_contributors, gather_coeffs, user_data );
3990+
3991+ STBIR_PROFILE_BUILD_START( cleanup );
3992+ stbir__cleanup_gathered_coefficients( samp->edge, &samp->extent_info, &samp->scale_info, gather_num_contributors, gather_contributors, gather_coeffs, gather_coefficient_width );
3993+ STBIR_PROFILE_BUILD_END( cleanup );
3994+
3995+ if ( !samp->is_gather )
3996+ {
3997+ // if this is a scatter (vertical only), then we need to pivot the coeffs
3998+ stbir__contributors * scatter_contributors;
3999+ int highest_set;
4000+
4001+ jump_right_to_pivot:
4002+
4003+ STBIR_PROFILE_BUILD_START( pivot );
4004+
4005+ highest_set = (-filter_pixel_margin) - 1;
4006+ for (n = 0; n < gather_num_contributors; n++)
4007+ {
4008+ int k;
4009+ int gn0 = gather_contributors->n0, gn1 = gather_contributors->n1;
4010+ int scatter_coefficient_width = samp->coefficient_width;
4011+ float * scatter_coeffs = samp->coefficients + ( gn0 + filter_pixel_margin ) * scatter_coefficient_width;
4012+ float * g_coeffs = gather_coeffs;
4013+ scatter_contributors = samp->contributors + ( gn0 + filter_pixel_margin );
4014+
4015+ for (k = gn0 ; k <= gn1 ; k++ )
4016+ {
4017+ float gc = *g_coeffs++;
4018+
4019+ // skip zero and denormals - must skip zeros to avoid adding coeffs beyond scatter_coefficient_width
4020+ // (which happens when pivoting from horizontal, which might have dummy zeros)
4021+ if ( ( ( gc >= stbir__small_float ) || ( gc <= -stbir__small_float ) ) )
4022+ {
4023+ if ( ( k > highest_set ) || ( scatter_contributors->n0 > scatter_contributors->n1 ) )
4024+ {
4025+ {
4026+ // if we are skipping over several contributors, we need to clear the skipped ones
4027+ stbir__contributors * clear_contributors = samp->contributors + ( highest_set + filter_pixel_margin + 1);
4028+ while ( clear_contributors < scatter_contributors )
4029+ {
4030+ clear_contributors->n0 = 0;
4031+ clear_contributors->n1 = -1;
4032+ ++clear_contributors;
4033+ }
4034+ }
4035+ scatter_contributors->n0 = n;
4036+ scatter_contributors->n1 = n;
4037+ scatter_coeffs[0] = gc;
4038+ highest_set = k;
4039+ }
4040+ else
4041+ {
4042+ stbir__insert_coeff( scatter_contributors, scatter_coeffs, n, gc, scatter_coefficient_width );
4043+ }
4044+ STBIR_ASSERT( ( scatter_contributors->n1 - scatter_contributors->n0 + 1 ) <= scatter_coefficient_width );
4045+ }
4046+ ++scatter_contributors;
4047+ scatter_coeffs += scatter_coefficient_width;
4048+ }
4049+
4050+ ++gather_contributors;
4051+ gather_coeffs += gather_coefficient_width;
4052+ }
4053+
4054+ // now clear any unset contribs
4055+ {
4056+ stbir__contributors * clear_contributors = samp->contributors + ( highest_set + filter_pixel_margin + 1);
4057+ stbir__contributors * end_contributors = samp->contributors + samp->num_contributors;
4058+ while ( clear_contributors < end_contributors )
4059+ {
4060+ clear_contributors->n0 = 0;
4061+ clear_contributors->n1 = -1;
4062+ ++clear_contributors;
4063+ }
4064+ }
4065+
4066+ STBIR_PROFILE_BUILD_END( pivot );
4067+ }
4068+ }
4069+ break;
4070+ }
4071+}
4072+
4073+
4074+//========================================================================================================
4075+// scanline decoders and encoders
4076+
4077+#define stbir__coder_min_num 1
4078+#define STB_IMAGE_RESIZE_DO_CODERS
4079+#include STBIR__HEADER_FILENAME
4080+
4081+#define stbir__decode_suffix BGRA
4082+#define stbir__decode_swizzle
4083+#define stbir__decode_order0 2
4084+#define stbir__decode_order1 1
4085+#define stbir__decode_order2 0
4086+#define stbir__decode_order3 3
4087+#define stbir__encode_order0 2
4088+#define stbir__encode_order1 1
4089+#define stbir__encode_order2 0
4090+#define stbir__encode_order3 3
4091+#define stbir__coder_min_num 4
4092+#define STB_IMAGE_RESIZE_DO_CODERS
4093+#include STBIR__HEADER_FILENAME
4094+
4095+#define stbir__decode_suffix ARGB
4096+#define stbir__decode_swizzle
4097+#define stbir__decode_order0 1
4098+#define stbir__decode_order1 2
4099+#define stbir__decode_order2 3
4100+#define stbir__decode_order3 0
4101+#define stbir__encode_order0 3
4102+#define stbir__encode_order1 0
4103+#define stbir__encode_order2 1
4104+#define stbir__encode_order3 2
4105+#define stbir__coder_min_num 4
4106+#define STB_IMAGE_RESIZE_DO_CODERS
4107+#include STBIR__HEADER_FILENAME
4108+
4109+#define stbir__decode_suffix ABGR
4110+#define stbir__decode_swizzle
4111+#define stbir__decode_order0 3
4112+#define stbir__decode_order1 2
4113+#define stbir__decode_order2 1
4114+#define stbir__decode_order3 0
4115+#define stbir__encode_order0 3
4116+#define stbir__encode_order1 2
4117+#define stbir__encode_order2 1
4118+#define stbir__encode_order3 0
4119+#define stbir__coder_min_num 4
4120+#define STB_IMAGE_RESIZE_DO_CODERS
4121+#include STBIR__HEADER_FILENAME
4122+
4123+#define stbir__decode_suffix AR
4124+#define stbir__decode_swizzle
4125+#define stbir__decode_order0 1
4126+#define stbir__decode_order1 0
4127+#define stbir__decode_order2 3
4128+#define stbir__decode_order3 2
4129+#define stbir__encode_order0 1
4130+#define stbir__encode_order1 0
4131+#define stbir__encode_order2 3
4132+#define stbir__encode_order3 2
4133+#define stbir__coder_min_num 2
4134+#define STB_IMAGE_RESIZE_DO_CODERS
4135+#include STBIR__HEADER_FILENAME
4136+
4137+
4138+// fancy alpha means we expand to keep both premultipied and non-premultiplied color channels
4139+static void stbir__fancy_alpha_weight_4ch( float * out_buffer, int width_times_channels )
4140+{
4141+ float STBIR_STREAMOUT_PTR(*) out = out_buffer;
4142+ float const * end_decode = out_buffer + ( width_times_channels / 4 ) * 7; // decode buffer aligned to end of out_buffer
4143+ float STBIR_STREAMOUT_PTR(*) decode = (float*)end_decode - width_times_channels;
4144+
4145+ // fancy alpha is stored internally as R G B A Rpm Gpm Bpm
4146+
4147+ #ifdef STBIR_SIMD
4148+
4149+ #ifdef STBIR_SIMD8
4150+ decode += 16;
4151+ STBIR_NO_UNROLL_LOOP_START
4152+ while ( decode <= end_decode )
4153+ {
4154+ stbir__simdf8 d0,d1,a0,a1,p0,p1;
4155+ STBIR_NO_UNROLL(decode);
4156+ stbir__simdf8_load( d0, decode-16 );
4157+ stbir__simdf8_load( d1, decode-16+8 );
4158+ stbir__simdf8_0123to33333333( a0, d0 );
4159+ stbir__simdf8_0123to33333333( a1, d1 );
4160+ stbir__simdf8_mult( p0, a0, d0 );
4161+ stbir__simdf8_mult( p1, a1, d1 );
4162+ stbir__simdf8_bot4s( a0, d0, p0 );
4163+ stbir__simdf8_bot4s( a1, d1, p1 );
4164+ stbir__simdf8_top4s( d0, d0, p0 );
4165+ stbir__simdf8_top4s( d1, d1, p1 );
4166+ stbir__simdf8_store ( out, a0 );
4167+ stbir__simdf8_store ( out+7, d0 );
4168+ stbir__simdf8_store ( out+14, a1 );
4169+ stbir__simdf8_store ( out+21, d1 );
4170+ decode += 16;
4171+ out += 28;
4172+ }
4173+ decode -= 16;
4174+ #else
4175+ decode += 8;
4176+ STBIR_NO_UNROLL_LOOP_START
4177+ while ( decode <= end_decode )
4178+ {
4179+ stbir__simdf d0,a0,d1,a1,p0,p1;
4180+ STBIR_NO_UNROLL(decode);
4181+ stbir__simdf_load( d0, decode-8 );
4182+ stbir__simdf_load( d1, decode-8+4 );
4183+ stbir__simdf_0123to3333( a0, d0 );
4184+ stbir__simdf_0123to3333( a1, d1 );
4185+ stbir__simdf_mult( p0, a0, d0 );
4186+ stbir__simdf_mult( p1, a1, d1 );
4187+ stbir__simdf_store ( out, d0 );
4188+ stbir__simdf_store ( out+4, p0 );
4189+ stbir__simdf_store ( out+7, d1 );
4190+ stbir__simdf_store ( out+7+4, p1 );
4191+ decode += 8;
4192+ out += 14;
4193+ }
4194+ decode -= 8;
4195+ #endif
4196+
4197+ // might be one last odd pixel
4198+ #ifdef STBIR_SIMD8
4199+ STBIR_NO_UNROLL_LOOP_START
4200+ while ( decode < end_decode )
4201+ #else
4202+ if ( decode < end_decode )
4203+ #endif
4204+ {
4205+ stbir__simdf d,a,p;
4206+ STBIR_NO_UNROLL(decode);
4207+ stbir__simdf_load( d, decode );
4208+ stbir__simdf_0123to3333( a, d );
4209+ stbir__simdf_mult( p, a, d );
4210+ stbir__simdf_store ( out, d );
4211+ stbir__simdf_store ( out+4, p );
4212+ decode += 4;
4213+ out += 7;
4214+ }
4215+
4216+ #else
4217+
4218+ while( decode < end_decode )
4219+ {
4220+ float r = decode[0], g = decode[1], b = decode[2], alpha = decode[3];
4221+ out[0] = r;
4222+ out[1] = g;
4223+ out[2] = b;
4224+ out[3] = alpha;
4225+ out[4] = r * alpha;
4226+ out[5] = g * alpha;
4227+ out[6] = b * alpha;
4228+ out += 7;
4229+ decode += 4;
4230+ }
4231+
4232+ #endif
4233+}
4234+
4235+static void stbir__fancy_alpha_weight_2ch( float * out_buffer, int width_times_channels )
4236+{
4237+ float STBIR_STREAMOUT_PTR(*) out = out_buffer;
4238+ float const * end_decode = out_buffer + ( width_times_channels / 2 ) * 3;
4239+ float STBIR_STREAMOUT_PTR(*) decode = (float*)end_decode - width_times_channels;
4240+
4241+ // for fancy alpha, turns into: [X A Xpm][X A Xpm],etc
4242+
4243+ #ifdef STBIR_SIMD
4244+
4245+ decode += 8;
4246+ if ( decode <= end_decode )
4247+ {
4248+ STBIR_NO_UNROLL_LOOP_START
4249+ do {
4250+ #ifdef STBIR_SIMD8
4251+ stbir__simdf8 d0,a0,p0;
4252+ STBIR_NO_UNROLL(decode);
4253+ stbir__simdf8_load( d0, decode-8 );
4254+ stbir__simdf8_0123to11331133( p0, d0 );
4255+ stbir__simdf8_0123to00220022( a0, d0 );
4256+ stbir__simdf8_mult( p0, p0, a0 );
4257+
4258+ stbir__simdf_store2( out, stbir__if_simdf8_cast_to_simdf4( d0 ) );
4259+ stbir__simdf_store( out+2, stbir__if_simdf8_cast_to_simdf4( p0 ) );
4260+ stbir__simdf_store2h( out+3, stbir__if_simdf8_cast_to_simdf4( d0 ) );
4261+
4262+ stbir__simdf_store2( out+6, stbir__simdf8_gettop4( d0 ) );
4263+ stbir__simdf_store( out+8, stbir__simdf8_gettop4( p0 ) );
4264+ stbir__simdf_store2h( out+9, stbir__simdf8_gettop4( d0 ) );
4265+ #else
4266+ stbir__simdf d0,a0,d1,a1,p0,p1;
4267+ STBIR_NO_UNROLL(decode);
4268+ stbir__simdf_load( d0, decode-8 );
4269+ stbir__simdf_load( d1, decode-8+4 );
4270+ stbir__simdf_0123to1133( p0, d0 );
4271+ stbir__simdf_0123to1133( p1, d1 );
4272+ stbir__simdf_0123to0022( a0, d0 );
4273+ stbir__simdf_0123to0022( a1, d1 );
4274+ stbir__simdf_mult( p0, p0, a0 );
4275+ stbir__simdf_mult( p1, p1, a1 );
4276+
4277+ stbir__simdf_store2( out, d0 );
4278+ stbir__simdf_store( out+2, p0 );
4279+ stbir__simdf_store2h( out+3, d0 );
4280+
4281+ stbir__simdf_store2( out+6, d1 );
4282+ stbir__simdf_store( out+8, p1 );
4283+ stbir__simdf_store2h( out+9, d1 );
4284+ #endif
4285+ decode += 8;
4286+ out += 12;
4287+ } while ( decode <= end_decode );
4288+ }
4289+ decode -= 8;
4290+ #endif
4291+
4292+ STBIR_SIMD_NO_UNROLL_LOOP_START
4293+ while( decode < end_decode )
4294+ {
4295+ float x = decode[0], y = decode[1];
4296+ STBIR_SIMD_NO_UNROLL(decode);
4297+ out[0] = x;
4298+ out[1] = y;
4299+ out[2] = x * y;
4300+ out += 3;
4301+ decode += 2;
4302+ }
4303+}
4304+
4305+static void stbir__fancy_alpha_unweight_4ch( float * encode_buffer, int width_times_channels )
4306+{
4307+ float STBIR_SIMD_STREAMOUT_PTR(*) encode = encode_buffer;
4308+ float STBIR_SIMD_STREAMOUT_PTR(*) input = encode_buffer;
4309+ float const * end_output = encode_buffer + width_times_channels;
4310+
4311+ // fancy RGBA is stored internally as R G B A Rpm Gpm Bpm
4312+
4313+ STBIR_SIMD_NO_UNROLL_LOOP_START
4314+ do {
4315+ float alpha = input[3];
4316+#ifdef STBIR_SIMD
4317+ stbir__simdf i,ia;
4318+ STBIR_SIMD_NO_UNROLL(encode);
4319+ if ( alpha < stbir__small_float )
4320+ {
4321+ stbir__simdf_load( i, input );
4322+ stbir__simdf_store( encode, i );
4323+ }
4324+ else
4325+ {
4326+ stbir__simdf_load1frep4( ia, 1.0f / alpha );
4327+ stbir__simdf_load( i, input+4 );
4328+ stbir__simdf_mult( i, i, ia );
4329+ stbir__simdf_store( encode, i );
4330+ encode[3] = alpha;
4331+ }
4332+#else
4333+ if ( alpha < stbir__small_float )
4334+ {
4335+ encode[0] = input[0];
4336+ encode[1] = input[1];
4337+ encode[2] = input[2];
4338+ }
4339+ else
4340+ {
4341+ float ialpha = 1.0f / alpha;
4342+ encode[0] = input[4] * ialpha;
4343+ encode[1] = input[5] * ialpha;
4344+ encode[2] = input[6] * ialpha;
4345+ }
4346+ encode[3] = alpha;
4347+#endif
4348+
4349+ input += 7;
4350+ encode += 4;
4351+ } while ( encode < end_output );
4352+}
4353+
4354+// format: [X A Xpm][X A Xpm] etc
4355+static void stbir__fancy_alpha_unweight_2ch( float * encode_buffer, int width_times_channels )
4356+{
4357+ float STBIR_SIMD_STREAMOUT_PTR(*) encode = encode_buffer;
4358+ float STBIR_SIMD_STREAMOUT_PTR(*) input = encode_buffer;
4359+ float const * end_output = encode_buffer + width_times_channels;
4360+
4361+ do {
4362+ float alpha = input[1];
4363+ encode[0] = input[0];
4364+ if ( alpha >= stbir__small_float )
4365+ encode[0] = input[2] / alpha;
4366+ encode[1] = alpha;
4367+
4368+ input += 3;
4369+ encode += 2;
4370+ } while ( encode < end_output );
4371+}
4372+
4373+static void stbir__simple_alpha_weight_4ch( float * decode_buffer, int width_times_channels )
4374+{
4375+ float STBIR_STREAMOUT_PTR(*) decode = decode_buffer;
4376+ float const * end_decode = decode_buffer + width_times_channels;
4377+
4378+ #ifdef STBIR_SIMD
4379+ {
4380+ decode += 2 * stbir__simdfX_float_count;
4381+ STBIR_NO_UNROLL_LOOP_START
4382+ while ( decode <= end_decode )
4383+ {
4384+ stbir__simdfX d0,a0,d1,a1;
4385+ STBIR_NO_UNROLL(decode);
4386+ stbir__simdfX_load( d0, decode-2*stbir__simdfX_float_count );
4387+ stbir__simdfX_load( d1, decode-2*stbir__simdfX_float_count+stbir__simdfX_float_count );
4388+ stbir__simdfX_aaa1( a0, d0, STBIR_onesX );
4389+ stbir__simdfX_aaa1( a1, d1, STBIR_onesX );
4390+ stbir__simdfX_mult( d0, d0, a0 );
4391+ stbir__simdfX_mult( d1, d1, a1 );
4392+ stbir__simdfX_store ( decode-2*stbir__simdfX_float_count, d0 );
4393+ stbir__simdfX_store ( decode-2*stbir__simdfX_float_count+stbir__simdfX_float_count, d1 );
4394+ decode += 2 * stbir__simdfX_float_count;
4395+ }
4396+ decode -= 2 * stbir__simdfX_float_count;
4397+
4398+ // few last pixels remnants
4399+ #ifdef STBIR_SIMD8
4400+ STBIR_NO_UNROLL_LOOP_START
4401+ while ( decode < end_decode )
4402+ #else
4403+ if ( decode < end_decode )
4404+ #endif
4405+ {
4406+ stbir__simdf d,a;
4407+ stbir__simdf_load( d, decode );
4408+ stbir__simdf_aaa1( a, d, STBIR__CONSTF(STBIR_ones) );
4409+ stbir__simdf_mult( d, d, a );
4410+ stbir__simdf_store ( decode, d );
4411+ decode += 4;
4412+ }
4413+ }
4414+
4415+ #else
4416+
4417+ while( decode < end_decode )
4418+ {
4419+ float alpha = decode[3];
4420+ decode[0] *= alpha;
4421+ decode[1] *= alpha;
4422+ decode[2] *= alpha;
4423+ decode += 4;
4424+ }
4425+
4426+ #endif
4427+}
4428+
4429+static void stbir__simple_alpha_weight_2ch( float * decode_buffer, int width_times_channels )
4430+{
4431+ float STBIR_STREAMOUT_PTR(*) decode = decode_buffer;
4432+ float const * end_decode = decode_buffer + width_times_channels;
4433+
4434+ #ifdef STBIR_SIMD
4435+ decode += 2 * stbir__simdfX_float_count;
4436+ STBIR_NO_UNROLL_LOOP_START
4437+ while ( decode <= end_decode )
4438+ {
4439+ stbir__simdfX d0,a0,d1,a1;
4440+ STBIR_NO_UNROLL(decode);
4441+ stbir__simdfX_load( d0, decode-2*stbir__simdfX_float_count );
4442+ stbir__simdfX_load( d1, decode-2*stbir__simdfX_float_count+stbir__simdfX_float_count );
4443+ stbir__simdfX_a1a1( a0, d0, STBIR_onesX );
4444+ stbir__simdfX_a1a1( a1, d1, STBIR_onesX );
4445+ stbir__simdfX_mult( d0, d0, a0 );
4446+ stbir__simdfX_mult( d1, d1, a1 );
4447+ stbir__simdfX_store ( decode-2*stbir__simdfX_float_count, d0 );
4448+ stbir__simdfX_store ( decode-2*stbir__simdfX_float_count+stbir__simdfX_float_count, d1 );
4449+ decode += 2 * stbir__simdfX_float_count;
4450+ }
4451+ decode -= 2 * stbir__simdfX_float_count;
4452+ #endif
4453+
4454+ STBIR_SIMD_NO_UNROLL_LOOP_START
4455+ while( decode < end_decode )
4456+ {
4457+ float alpha = decode[1];
4458+ STBIR_SIMD_NO_UNROLL(decode);
4459+ decode[0] *= alpha;
4460+ decode += 2;
4461+ }
4462+}
4463+
4464+static void stbir__simple_alpha_unweight_4ch( float * encode_buffer, int width_times_channels )
4465+{
4466+ float STBIR_SIMD_STREAMOUT_PTR(*) encode = encode_buffer;
4467+ float const * end_output = encode_buffer + width_times_channels;
4468+
4469+ STBIR_SIMD_NO_UNROLL_LOOP_START
4470+ do {
4471+ float alpha = encode[3];
4472+
4473+#ifdef STBIR_SIMD
4474+ stbir__simdf i,ia;
4475+ STBIR_SIMD_NO_UNROLL(encode);
4476+ if ( alpha >= stbir__small_float )
4477+ {
4478+ stbir__simdf_load1frep4( ia, 1.0f / alpha );
4479+ stbir__simdf_load( i, encode );
4480+ stbir__simdf_mult( i, i, ia );
4481+ stbir__simdf_store( encode, i );
4482+ encode[3] = alpha;
4483+ }
4484+#else
4485+ if ( alpha >= stbir__small_float )
4486+ {
4487+ float ialpha = 1.0f / alpha;
4488+ encode[0] *= ialpha;
4489+ encode[1] *= ialpha;
4490+ encode[2] *= ialpha;
4491+ }
4492+#endif
4493+ encode += 4;
4494+ } while ( encode < end_output );
4495+}
4496+
4497+static void stbir__simple_alpha_unweight_2ch( float * encode_buffer, int width_times_channels )
4498+{
4499+ float STBIR_SIMD_STREAMOUT_PTR(*) encode = encode_buffer;
4500+ float const * end_output = encode_buffer + width_times_channels;
4501+
4502+ do {
4503+ float alpha = encode[1];
4504+ if ( alpha >= stbir__small_float )
4505+ encode[0] /= alpha;
4506+ encode += 2;
4507+ } while ( encode < end_output );
4508+}
4509+
4510+
4511+// only used in RGB->BGR or BGR->RGB
4512+static void stbir__simple_flip_3ch( float * decode_buffer, int width_times_channels )
4513+{
4514+ float STBIR_STREAMOUT_PTR(*) decode = decode_buffer;
4515+ float const * end_decode = decode_buffer + width_times_channels;
4516+
4517+#ifdef STBIR_SIMD
4518+ #ifdef stbir__simdf_swiz2 // do we have two argument swizzles?
4519+ end_decode -= 12;
4520+ STBIR_NO_UNROLL_LOOP_START
4521+ while( decode <= end_decode )
4522+ {
4523+ // on arm64 8 instructions, no overlapping stores
4524+ stbir__simdf a,b,c,na,nb;
4525+ STBIR_SIMD_NO_UNROLL(decode);
4526+ stbir__simdf_load( a, decode );
4527+ stbir__simdf_load( b, decode+4 );
4528+ stbir__simdf_load( c, decode+8 );
4529+
4530+ na = stbir__simdf_swiz2( a, b, 2, 1, 0, 5 );
4531+ b = stbir__simdf_swiz2( a, b, 4, 3, 6, 7 );
4532+ nb = stbir__simdf_swiz2( b, c, 0, 1, 4, 3 );
4533+ c = stbir__simdf_swiz2( b, c, 2, 7, 6, 5 );
4534+
4535+ stbir__simdf_store( decode, na );
4536+ stbir__simdf_store( decode+4, nb );
4537+ stbir__simdf_store( decode+8, c );
4538+ decode += 12;
4539+ }
4540+ end_decode += 12;
4541+ #else
4542+ end_decode -= 24;
4543+ STBIR_NO_UNROLL_LOOP_START
4544+ while( decode <= end_decode )
4545+ {
4546+ // 26 instructions on x64
4547+ stbir__simdf a,b,c,d,e,f,g;
4548+ float i21, i23;
4549+ STBIR_SIMD_NO_UNROLL(decode);
4550+ stbir__simdf_load( a, decode );
4551+ stbir__simdf_load( b, decode+3 );
4552+ stbir__simdf_load( c, decode+6 );
4553+ stbir__simdf_load( d, decode+9 );
4554+ stbir__simdf_load( e, decode+12 );
4555+ stbir__simdf_load( f, decode+15 );
4556+ stbir__simdf_load( g, decode+18 );
4557+
4558+ a = stbir__simdf_swiz( a, 2, 1, 0, 3 );
4559+ b = stbir__simdf_swiz( b, 2, 1, 0, 3 );
4560+ c = stbir__simdf_swiz( c, 2, 1, 0, 3 );
4561+ d = stbir__simdf_swiz( d, 2, 1, 0, 3 );
4562+ e = stbir__simdf_swiz( e, 2, 1, 0, 3 );
4563+ f = stbir__simdf_swiz( f, 2, 1, 0, 3 );
4564+ g = stbir__simdf_swiz( g, 2, 1, 0, 3 );
4565+
4566+ // stores overlap, need to be in order,
4567+ stbir__simdf_store( decode, a );
4568+ i21 = decode[21];
4569+ stbir__simdf_store( decode+3, b );
4570+ i23 = decode[23];
4571+ stbir__simdf_store( decode+6, c );
4572+ stbir__simdf_store( decode+9, d );
4573+ stbir__simdf_store( decode+12, e );
4574+ stbir__simdf_store( decode+15, f );
4575+ stbir__simdf_store( decode+18, g );
4576+ decode[21] = i23;
4577+ decode[23] = i21;
4578+ decode += 24;
4579+ }
4580+ end_decode += 24;
4581+ #endif
4582+#else
4583+ end_decode -= 12;
4584+ STBIR_NO_UNROLL_LOOP_START
4585+ while( decode <= end_decode )
4586+ {
4587+ // 16 instructions
4588+ float t0,t1,t2,t3;
4589+ STBIR_NO_UNROLL(decode);
4590+ t0 = decode[0]; t1 = decode[3]; t2 = decode[6]; t3 = decode[9];
4591+ decode[0] = decode[2]; decode[3] = decode[5]; decode[6] = decode[8]; decode[9] = decode[11];
4592+ decode[2] = t0; decode[5] = t1; decode[8] = t2; decode[11] = t3;
4593+ decode += 12;
4594+ }
4595+ end_decode += 12;
4596+#endif
4597+
4598+ STBIR_NO_UNROLL_LOOP_START
4599+ while( decode < end_decode )
4600+ {
4601+ float t = decode[0];
4602+ STBIR_NO_UNROLL(decode);
4603+ decode[0] = decode[2];
4604+ decode[2] = t;
4605+ decode += 3;
4606+ }
4607+}
4608+
4609+
4610+
4611+static void stbir__decode_scanline(stbir__info const * stbir_info, int n, float * output_buffer STBIR_ONLY_PROFILE_GET_SPLIT_INFO )
4612+{
4613+ int channels = stbir_info->channels;
4614+ int effective_channels = stbir_info->effective_channels;
4615+ int input_sample_in_bytes = stbir__type_size[stbir_info->input_type] * channels;
4616+ stbir_edge edge_horizontal = stbir_info->horizontal.edge;
4617+ stbir_edge edge_vertical = stbir_info->vertical.edge;
4618+ int row = stbir__edge_wrap(edge_vertical, n, stbir_info->vertical.scale_info.input_full_size);
4619+ const void* input_plane_data = ( (char *) stbir_info->input_data ) + (size_t)row * (size_t) stbir_info->input_stride_bytes;
4620+ stbir__span const * spans = stbir_info->scanline_extents.spans;
4621+ float * full_decode_buffer = output_buffer - stbir_info->scanline_extents.conservative.n0 * effective_channels;
4622+ float * last_decoded = 0;
4623+
4624+ // if we are on edge_zero, and we get in here with an out of bounds n, then the calculate filters has failed
4625+ STBIR_ASSERT( !(edge_vertical == STBIR_EDGE_ZERO && (n < 0 || n >= stbir_info->vertical.scale_info.input_full_size)) );
4626+
4627+ do
4628+ {
4629+ float * decode_buffer;
4630+ void const * input_data;
4631+ float * end_decode;
4632+ int width_times_channels;
4633+ int width;
4634+
4635+ if ( spans->n1 < spans->n0 )
4636+ break;
4637+
4638+ width = spans->n1 + 1 - spans->n0;
4639+ decode_buffer = full_decode_buffer + spans->n0 * effective_channels;
4640+ end_decode = full_decode_buffer + ( spans->n1 + 1 ) * effective_channels;
4641+ width_times_channels = width * channels;
4642+
4643+ // read directly out of input plane by default
4644+ input_data = ( (char*)input_plane_data ) + spans->pixel_offset_for_input * input_sample_in_bytes;
4645+
4646+ // if we have an input callback, call it to get the input data
4647+ if ( stbir_info->in_pixels_cb )
4648+ {
4649+ // call the callback with a temp buffer (that they can choose to use or not). the temp is just right aligned memory in the decode_buffer itself
4650+ input_data = stbir_info->in_pixels_cb( ( (char*) end_decode ) - ( width * input_sample_in_bytes ) + ( ( stbir_info->input_type != STBIR_TYPE_FLOAT ) ? ( sizeof(float)*STBIR_INPUT_CALLBACK_PADDING ) : 0 ), input_plane_data, width, spans->pixel_offset_for_input, row, stbir_info->user_data );
4651+ }
4652+
4653+ STBIR_PROFILE_START( decode );
4654+ // convert the pixels info the float decode_buffer, (we index from end_decode, so that when channels<effective_channels, we are right justified in the buffer)
4655+ last_decoded = stbir_info->decode_pixels( (float*)end_decode - width_times_channels, width_times_channels, input_data );
4656+ STBIR_PROFILE_END( decode );
4657+
4658+ if (stbir_info->alpha_weight)
4659+ {
4660+ STBIR_PROFILE_START( alpha );
4661+ stbir_info->alpha_weight( decode_buffer, width_times_channels );
4662+ STBIR_PROFILE_END( alpha );
4663+ }
4664+
4665+ ++spans;
4666+ } while ( spans <= ( &stbir_info->scanline_extents.spans[1] ) );
4667+
4668+ // handle the edge_wrap filter (all other types are handled back out at the calculate_filter stage)
4669+ // basically the idea here is that if we have the whole scanline in memory, we don't redecode the
4670+ // wrapped edge pixels, and instead just memcpy them from the scanline into the edge positions
4671+ if ( ( edge_horizontal == STBIR_EDGE_WRAP ) && ( stbir_info->scanline_extents.edge_sizes[0] | stbir_info->scanline_extents.edge_sizes[1] ) )
4672+ {
4673+ // this code only runs if we're in edge_wrap, and we're doing the entire scanline
4674+ int e, start_x[2];
4675+ int input_full_size = stbir_info->horizontal.scale_info.input_full_size;
4676+
4677+ start_x[0] = -stbir_info->scanline_extents.edge_sizes[0]; // left edge start x
4678+ start_x[1] = input_full_size; // right edge
4679+
4680+ for( e = 0; e < 2 ; e++ )
4681+ {
4682+ // do each margin
4683+ int margin = stbir_info->scanline_extents.edge_sizes[e];
4684+ if ( margin )
4685+ {
4686+ int x = start_x[e];
4687+ float * marg = full_decode_buffer + x * effective_channels;
4688+ float const * src = full_decode_buffer + stbir__edge_wrap(edge_horizontal, x, input_full_size) * effective_channels;
4689+ STBIR_MEMCPY( marg, src, margin * effective_channels * sizeof(float) );
4690+ if ( e == 1 ) last_decoded = marg + margin * effective_channels;
4691+ }
4692+ }
4693+ }
4694+
4695+ // some of the horizontal gathers read one float off the edge (which is masked out), but we force a zero here to make sure no NaNs leak in
4696+ // (we can't pre-zero it, because the input callback can use that area as padding)
4697+ last_decoded[0] = 0.0f;
4698+
4699+ // we clear this extra float, because the final output pixel filter kernel might have used one less coeff than the max filter width
4700+ // when this happens, we do read that pixel from the input, so it too could be Nan, so just zero an extra one.
4701+ // this fits because each scanline is padded by three floats (STBIR_INPUT_CALLBACK_PADDING)
4702+ last_decoded[1] = 0.0f;
4703+}
4704+
4705+
4706+//=================
4707+// Do 1 channel horizontal routines
4708+
4709+#ifdef STBIR_SIMD
4710+
4711+#define stbir__1_coeff_only() \
4712+ stbir__simdf tot,c; \
4713+ STBIR_SIMD_NO_UNROLL(decode); \
4714+ stbir__simdf_load1( c, hc ); \
4715+ stbir__simdf_mult1_mem( tot, c, decode );
4716+
4717+#define stbir__2_coeff_only() \
4718+ stbir__simdf tot,c,d; \
4719+ STBIR_SIMD_NO_UNROLL(decode); \
4720+ stbir__simdf_load2z( c, hc ); \
4721+ stbir__simdf_load2( d, decode ); \
4722+ stbir__simdf_mult( tot, c, d ); \
4723+ stbir__simdf_0123to1230( c, tot ); \
4724+ stbir__simdf_add1( tot, tot, c );
4725+
4726+#define stbir__3_coeff_only() \
4727+ stbir__simdf tot,c,t; \
4728+ STBIR_SIMD_NO_UNROLL(decode); \
4729+ stbir__simdf_load( c, hc ); \
4730+ stbir__simdf_mult_mem( tot, c, decode ); \
4731+ stbir__simdf_0123to1230( c, tot ); \
4732+ stbir__simdf_0123to2301( t, tot ); \
4733+ stbir__simdf_add1( tot, tot, c ); \
4734+ stbir__simdf_add1( tot, tot, t );
4735+
4736+#define stbir__store_output_tiny() \
4737+ stbir__simdf_store1( output, tot ); \
4738+ horizontal_coefficients += coefficient_width; \
4739+ ++horizontal_contributors; \
4740+ output += 1;
4741+
4742+#define stbir__4_coeff_start() \
4743+ stbir__simdf tot,c; \
4744+ STBIR_SIMD_NO_UNROLL(decode); \
4745+ stbir__simdf_load( c, hc ); \
4746+ stbir__simdf_mult_mem( tot, c, decode ); \
4747+
4748+#define stbir__4_coeff_continue_from_4( ofs ) \
4749+ STBIR_SIMD_NO_UNROLL(decode); \
4750+ stbir__simdf_load( c, hc + (ofs) ); \
4751+ stbir__simdf_madd_mem( tot, tot, c, decode+(ofs) );
4752+
4753+#define stbir__1_coeff_remnant( ofs ) \
4754+ { stbir__simdf d; \
4755+ stbir__simdf_load1z( c, hc + (ofs) ); \
4756+ stbir__simdf_load1( d, decode + (ofs) ); \
4757+ stbir__simdf_madd( tot, tot, d, c ); }
4758+
4759+#define stbir__2_coeff_remnant( ofs ) \
4760+ { stbir__simdf d; \
4761+ stbir__simdf_load2z( c, hc+(ofs) ); \
4762+ stbir__simdf_load2( d, decode+(ofs) ); \
4763+ stbir__simdf_madd( tot, tot, d, c ); }
4764+
4765+#define stbir__3_coeff_setup() \
4766+ stbir__simdf mask; \
4767+ stbir__simdf_load( mask, STBIR_mask + 3 );
4768+
4769+#define stbir__3_coeff_remnant( ofs ) \
4770+ stbir__simdf_load( c, hc+(ofs) ); \
4771+ stbir__simdf_and( c, c, mask ); \
4772+ stbir__simdf_madd_mem( tot, tot, c, decode+(ofs) );
4773+
4774+#define stbir__store_output() \
4775+ stbir__simdf_0123to2301( c, tot ); \
4776+ stbir__simdf_add( tot, tot, c ); \
4777+ stbir__simdf_0123to1230( c, tot ); \
4778+ stbir__simdf_add1( tot, tot, c ); \
4779+ stbir__simdf_store1( output, tot ); \
4780+ horizontal_coefficients += coefficient_width; \
4781+ ++horizontal_contributors; \
4782+ output += 1;
4783+
4784+#else
4785+
4786+#define stbir__1_coeff_only() \
4787+ float tot; \
4788+ tot = decode[0]*hc[0];
4789+
4790+#define stbir__2_coeff_only() \
4791+ float tot; \
4792+ tot = decode[0] * hc[0]; \
4793+ tot += decode[1] * hc[1];
4794+
4795+#define stbir__3_coeff_only() \
4796+ float tot; \
4797+ tot = decode[0] * hc[0]; \
4798+ tot += decode[1] * hc[1]; \
4799+ tot += decode[2] * hc[2];
4800+
4801+#define stbir__store_output_tiny() \
4802+ output[0] = tot; \
4803+ horizontal_coefficients += coefficient_width; \
4804+ ++horizontal_contributors; \
4805+ output += 1;
4806+
4807+#define stbir__4_coeff_start() \
4808+ float tot0,tot1,tot2,tot3; \
4809+ tot0 = decode[0] * hc[0]; \
4810+ tot1 = decode[1] * hc[1]; \
4811+ tot2 = decode[2] * hc[2]; \
4812+ tot3 = decode[3] * hc[3];
4813+
4814+#define stbir__4_coeff_continue_from_4( ofs ) \
4815+ tot0 += decode[0+(ofs)] * hc[0+(ofs)]; \
4816+ tot1 += decode[1+(ofs)] * hc[1+(ofs)]; \
4817+ tot2 += decode[2+(ofs)] * hc[2+(ofs)]; \
4818+ tot3 += decode[3+(ofs)] * hc[3+(ofs)];
4819+
4820+#define stbir__1_coeff_remnant( ofs ) \
4821+ tot0 += decode[0+(ofs)] * hc[0+(ofs)];
4822+
4823+#define stbir__2_coeff_remnant( ofs ) \
4824+ tot0 += decode[0+(ofs)] * hc[0+(ofs)]; \
4825+ tot1 += decode[1+(ofs)] * hc[1+(ofs)]; \
4826+
4827+#define stbir__3_coeff_remnant( ofs ) \
4828+ tot0 += decode[0+(ofs)] * hc[0+(ofs)]; \
4829+ tot1 += decode[1+(ofs)] * hc[1+(ofs)]; \
4830+ tot2 += decode[2+(ofs)] * hc[2+(ofs)];
4831+
4832+#define stbir__store_output() \
4833+ output[0] = (tot0+tot2)+(tot1+tot3); \
4834+ horizontal_coefficients += coefficient_width; \
4835+ ++horizontal_contributors; \
4836+ output += 1;
4837+
4838+#endif
4839+
4840+#define STBIR__horizontal_channels 1
4841+#define STB_IMAGE_RESIZE_DO_HORIZONTALS
4842+#include STBIR__HEADER_FILENAME
4843+
4844+
4845+//=================
4846+// Do 2 channel horizontal routines
4847+
4848+#ifdef STBIR_SIMD
4849+
4850+#define stbir__1_coeff_only() \
4851+ stbir__simdf tot,c,d; \
4852+ STBIR_SIMD_NO_UNROLL(decode); \
4853+ stbir__simdf_load1z( c, hc ); \
4854+ stbir__simdf_0123to0011( c, c ); \
4855+ stbir__simdf_load2( d, decode ); \
4856+ stbir__simdf_mult( tot, d, c );
4857+
4858+#define stbir__2_coeff_only() \
4859+ stbir__simdf tot,c; \
4860+ STBIR_SIMD_NO_UNROLL(decode); \
4861+ stbir__simdf_load2( c, hc ); \
4862+ stbir__simdf_0123to0011( c, c ); \
4863+ stbir__simdf_mult_mem( tot, c, decode );
4864+
4865+#define stbir__3_coeff_only() \
4866+ stbir__simdf tot,c,cs,d; \
4867+ STBIR_SIMD_NO_UNROLL(decode); \
4868+ stbir__simdf_load( cs, hc ); \
4869+ stbir__simdf_0123to0011( c, cs ); \
4870+ stbir__simdf_mult_mem( tot, c, decode ); \
4871+ stbir__simdf_0123to2222( c, cs ); \
4872+ stbir__simdf_load2z( d, decode+4 ); \
4873+ stbir__simdf_madd( tot, tot, d, c );
4874+
4875+#define stbir__store_output_tiny() \
4876+ stbir__simdf_0123to2301( c, tot ); \
4877+ stbir__simdf_add( tot, tot, c ); \
4878+ stbir__simdf_store2( output, tot ); \
4879+ horizontal_coefficients += coefficient_width; \
4880+ ++horizontal_contributors; \
4881+ output += 2;
4882+
4883+#ifdef STBIR_SIMD8
4884+
4885+#define stbir__4_coeff_start() \
4886+ stbir__simdf8 tot0,c,cs; \
4887+ STBIR_SIMD_NO_UNROLL(decode); \
4888+ stbir__simdf8_load4b( cs, hc ); \
4889+ stbir__simdf8_0123to00112233( c, cs ); \
4890+ stbir__simdf8_mult_mem( tot0, c, decode );
4891+
4892+#define stbir__4_coeff_continue_from_4( ofs ) \
4893+ STBIR_SIMD_NO_UNROLL(decode); \
4894+ stbir__simdf8_load4b( cs, hc + (ofs) ); \
4895+ stbir__simdf8_0123to00112233( c, cs ); \
4896+ stbir__simdf8_madd_mem( tot0, tot0, c, decode+(ofs)*2 );
4897+
4898+#define stbir__1_coeff_remnant( ofs ) \
4899+ { stbir__simdf t,d; \
4900+ stbir__simdf_load1z( t, hc + (ofs) ); \
4901+ stbir__simdf_load2( d, decode + (ofs) * 2 ); \
4902+ stbir__simdf_0123to0011( t, t ); \
4903+ stbir__simdf_mult( t, t, d ); \
4904+ stbir__simdf8_add4( tot0, tot0, t ); }
4905+
4906+#define stbir__2_coeff_remnant( ofs ) \
4907+ { stbir__simdf t; \
4908+ stbir__simdf_load2( t, hc + (ofs) ); \
4909+ stbir__simdf_0123to0011( t, t ); \
4910+ stbir__simdf_mult_mem( t, t, decode+(ofs)*2 ); \
4911+ stbir__simdf8_add4( tot0, tot0, t ); }
4912+
4913+#define stbir__3_coeff_remnant( ofs ) \
4914+ { stbir__simdf8 d; \
4915+ stbir__simdf8_load4b( cs, hc + (ofs) ); \
4916+ stbir__simdf8_0123to00112233( c, cs ); \
4917+ stbir__simdf8_load6z( d, decode+(ofs)*2 ); \
4918+ stbir__simdf8_madd( tot0, tot0, c, d ); }
4919+
4920+#define stbir__store_output() \
4921+ { stbir__simdf t,d; \
4922+ stbir__simdf8_add4halves( t, stbir__if_simdf8_cast_to_simdf4(tot0), tot0 ); \
4923+ stbir__simdf_0123to2301( d, t ); \
4924+ stbir__simdf_add( t, t, d ); \
4925+ stbir__simdf_store2( output, t ); \
4926+ horizontal_coefficients += coefficient_width; \
4927+ ++horizontal_contributors; \
4928+ output += 2; }
4929+
4930+#else
4931+
4932+#define stbir__4_coeff_start() \
4933+ stbir__simdf tot0,tot1,c,cs; \
4934+ STBIR_SIMD_NO_UNROLL(decode); \
4935+ stbir__simdf_load( cs, hc ); \
4936+ stbir__simdf_0123to0011( c, cs ); \
4937+ stbir__simdf_mult_mem( tot0, c, decode ); \
4938+ stbir__simdf_0123to2233( c, cs ); \
4939+ stbir__simdf_mult_mem( tot1, c, decode+4 );
4940+
4941+#define stbir__4_coeff_continue_from_4( ofs ) \
4942+ STBIR_SIMD_NO_UNROLL(decode); \
4943+ stbir__simdf_load( cs, hc + (ofs) ); \
4944+ stbir__simdf_0123to0011( c, cs ); \
4945+ stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*2 ); \
4946+ stbir__simdf_0123to2233( c, cs ); \
4947+ stbir__simdf_madd_mem( tot1, tot1, c, decode+(ofs)*2+4 );
4948+
4949+#define stbir__1_coeff_remnant( ofs ) \
4950+ { stbir__simdf d; \
4951+ stbir__simdf_load1z( cs, hc + (ofs) ); \
4952+ stbir__simdf_0123to0011( c, cs ); \
4953+ stbir__simdf_load2( d, decode + (ofs) * 2 ); \
4954+ stbir__simdf_madd( tot0, tot0, d, c ); }
4955+
4956+#define stbir__2_coeff_remnant( ofs ) \
4957+ stbir__simdf_load2( cs, hc + (ofs) ); \
4958+ stbir__simdf_0123to0011( c, cs ); \
4959+ stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*2 );
4960+
4961+#define stbir__3_coeff_remnant( ofs ) \
4962+ { stbir__simdf d; \
4963+ stbir__simdf_load( cs, hc + (ofs) ); \
4964+ stbir__simdf_0123to0011( c, cs ); \
4965+ stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*2 ); \
4966+ stbir__simdf_0123to2222( c, cs ); \
4967+ stbir__simdf_load2z( d, decode + (ofs) * 2 + 4 ); \
4968+ stbir__simdf_madd( tot1, tot1, d, c ); }
4969+
4970+#define stbir__store_output() \
4971+ stbir__simdf_add( tot0, tot0, tot1 ); \
4972+ stbir__simdf_0123to2301( c, tot0 ); \
4973+ stbir__simdf_add( tot0, tot0, c ); \
4974+ stbir__simdf_store2( output, tot0 ); \
4975+ horizontal_coefficients += coefficient_width; \
4976+ ++horizontal_contributors; \
4977+ output += 2;
4978+
4979+#endif
4980+
4981+#else
4982+
4983+#define stbir__1_coeff_only() \
4984+ float tota,totb,c; \
4985+ c = hc[0]; \
4986+ tota = decode[0]*c; \
4987+ totb = decode[1]*c;
4988+
4989+#define stbir__2_coeff_only() \
4990+ float tota,totb,c; \
4991+ c = hc[0]; \
4992+ tota = decode[0]*c; \
4993+ totb = decode[1]*c; \
4994+ c = hc[1]; \
4995+ tota += decode[2]*c; \
4996+ totb += decode[3]*c;
4997+
4998+// this weird order of add matches the simd
4999+#define stbir__3_coeff_only() \
5000+ float tota,totb,c; \
5001+ c = hc[0]; \
5002+ tota = decode[0]*c; \
5003+ totb = decode[1]*c; \
5004+ c = hc[2]; \
5005+ tota += decode[4]*c; \
5006+ totb += decode[5]*c; \
5007+ c = hc[1]; \
5008+ tota += decode[2]*c; \
5009+ totb += decode[3]*c;
5010+
5011+#define stbir__store_output_tiny() \
5012+ output[0] = tota; \
5013+ output[1] = totb; \
5014+ horizontal_coefficients += coefficient_width; \
5015+ ++horizontal_contributors; \
5016+ output += 2;
5017+
5018+#define stbir__4_coeff_start() \
5019+ float tota0,tota1,tota2,tota3,totb0,totb1,totb2,totb3,c; \
5020+ c = hc[0]; \
5021+ tota0 = decode[0]*c; \
5022+ totb0 = decode[1]*c; \
5023+ c = hc[1]; \
5024+ tota1 = decode[2]*c; \
5025+ totb1 = decode[3]*c; \
5026+ c = hc[2]; \
5027+ tota2 = decode[4]*c; \
5028+ totb2 = decode[5]*c; \
5029+ c = hc[3]; \
5030+ tota3 = decode[6]*c; \
5031+ totb3 = decode[7]*c;
5032+
5033+#define stbir__4_coeff_continue_from_4( ofs ) \
5034+ c = hc[0+(ofs)]; \
5035+ tota0 += decode[0+(ofs)*2]*c; \
5036+ totb0 += decode[1+(ofs)*2]*c; \
5037+ c = hc[1+(ofs)]; \
5038+ tota1 += decode[2+(ofs)*2]*c; \
5039+ totb1 += decode[3+(ofs)*2]*c; \
5040+ c = hc[2+(ofs)]; \
5041+ tota2 += decode[4+(ofs)*2]*c; \
5042+ totb2 += decode[5+(ofs)*2]*c; \
5043+ c = hc[3+(ofs)]; \
5044+ tota3 += decode[6+(ofs)*2]*c; \
5045+ totb3 += decode[7+(ofs)*2]*c;
5046+
5047+#define stbir__1_coeff_remnant( ofs ) \
5048+ c = hc[0+(ofs)]; \
5049+ tota0 += decode[0+(ofs)*2] * c; \
5050+ totb0 += decode[1+(ofs)*2] * c;
5051+
5052+#define stbir__2_coeff_remnant( ofs ) \
5053+ c = hc[0+(ofs)]; \
5054+ tota0 += decode[0+(ofs)*2] * c; \
5055+ totb0 += decode[1+(ofs)*2] * c; \
5056+ c = hc[1+(ofs)]; \
5057+ tota1 += decode[2+(ofs)*2] * c; \
5058+ totb1 += decode[3+(ofs)*2] * c;
5059+
5060+#define stbir__3_coeff_remnant( ofs ) \
5061+ c = hc[0+(ofs)]; \
5062+ tota0 += decode[0+(ofs)*2] * c; \
5063+ totb0 += decode[1+(ofs)*2] * c; \
5064+ c = hc[1+(ofs)]; \
5065+ tota1 += decode[2+(ofs)*2] * c; \
5066+ totb1 += decode[3+(ofs)*2] * c; \
5067+ c = hc[2+(ofs)]; \
5068+ tota2 += decode[4+(ofs)*2] * c; \
5069+ totb2 += decode[5+(ofs)*2] * c;
5070+
5071+#define stbir__store_output() \
5072+ output[0] = (tota0+tota2)+(tota1+tota3); \
5073+ output[1] = (totb0+totb2)+(totb1+totb3); \
5074+ horizontal_coefficients += coefficient_width; \
5075+ ++horizontal_contributors; \
5076+ output += 2;
5077+
5078+#endif
5079+
5080+#define STBIR__horizontal_channels 2
5081+#define STB_IMAGE_RESIZE_DO_HORIZONTALS
5082+#include STBIR__HEADER_FILENAME
5083+
5084+
5085+//=================
5086+// Do 3 channel horizontal routines
5087+
5088+#ifdef STBIR_SIMD
5089+
5090+#define stbir__1_coeff_only() \
5091+ stbir__simdf tot,c,d; \
5092+ STBIR_SIMD_NO_UNROLL(decode); \
5093+ stbir__simdf_load1z( c, hc ); \
5094+ stbir__simdf_0123to0001( c, c ); \
5095+ stbir__simdf_load( d, decode ); \
5096+ stbir__simdf_mult( tot, d, c );
5097+
5098+#define stbir__2_coeff_only() \
5099+ stbir__simdf tot,c,cs,d; \
5100+ STBIR_SIMD_NO_UNROLL(decode); \
5101+ stbir__simdf_load2( cs, hc ); \
5102+ stbir__simdf_0123to0000( c, cs ); \
5103+ stbir__simdf_load( d, decode ); \
5104+ stbir__simdf_mult( tot, d, c ); \
5105+ stbir__simdf_0123to1111( c, cs ); \
5106+ stbir__simdf_load( d, decode+3 ); \
5107+ stbir__simdf_madd( tot, tot, d, c );
5108+
5109+#define stbir__3_coeff_only() \
5110+ stbir__simdf tot,c,d,cs; \
5111+ STBIR_SIMD_NO_UNROLL(decode); \
5112+ stbir__simdf_load( cs, hc ); \
5113+ stbir__simdf_0123to0000( c, cs ); \
5114+ stbir__simdf_load( d, decode ); \
5115+ stbir__simdf_mult( tot, d, c ); \
5116+ stbir__simdf_0123to1111( c, cs ); \
5117+ stbir__simdf_load( d, decode+3 ); \
5118+ stbir__simdf_madd( tot, tot, d, c ); \
5119+ stbir__simdf_0123to2222( c, cs ); \
5120+ stbir__simdf_load( d, decode+6 ); \
5121+ stbir__simdf_madd( tot, tot, d, c );
5122+
5123+#define stbir__store_output_tiny() \
5124+ stbir__simdf_store2( output, tot ); \
5125+ stbir__simdf_0123to2301( tot, tot ); \
5126+ stbir__simdf_store1( output+2, tot ); \
5127+ horizontal_coefficients += coefficient_width; \
5128+ ++horizontal_contributors; \
5129+ output += 3;
5130+
5131+#ifdef STBIR_SIMD8
5132+
5133+// we're loading from the XXXYYY decode by -1 to get the XXXYYY into different halves of the AVX reg fyi
5134+#define stbir__4_coeff_start() \
5135+ stbir__simdf8 tot0,tot1,c,cs; stbir__simdf t; \
5136+ STBIR_SIMD_NO_UNROLL(decode); \
5137+ stbir__simdf8_load4b( cs, hc ); \
5138+ stbir__simdf8_0123to00001111( c, cs ); \
5139+ stbir__simdf8_mult_mem( tot0, c, decode - 1 ); \
5140+ stbir__simdf8_0123to22223333( c, cs ); \
5141+ stbir__simdf8_mult_mem( tot1, c, decode+6 - 1 );
5142+
5143+#define stbir__4_coeff_continue_from_4( ofs ) \
5144+ STBIR_SIMD_NO_UNROLL(decode); \
5145+ stbir__simdf8_load4b( cs, hc + (ofs) ); \
5146+ stbir__simdf8_0123to00001111( c, cs ); \
5147+ stbir__simdf8_madd_mem( tot0, tot0, c, decode+(ofs)*3 - 1 ); \
5148+ stbir__simdf8_0123to22223333( c, cs ); \
5149+ stbir__simdf8_madd_mem( tot1, tot1, c, decode+(ofs)*3 + 6 - 1 );
5150+
5151+#define stbir__1_coeff_remnant( ofs ) \
5152+ STBIR_SIMD_NO_UNROLL(decode); \
5153+ stbir__simdf_load1rep4( t, hc + (ofs) ); \
5154+ stbir__simdf8_madd_mem4( tot0, tot0, t, decode+(ofs)*3 - 1 );
5155+
5156+#define stbir__2_coeff_remnant( ofs ) \
5157+ STBIR_SIMD_NO_UNROLL(decode); \
5158+ stbir__simdf8_load4b( cs, hc + (ofs) - 2 ); \
5159+ stbir__simdf8_0123to22223333( c, cs ); \
5160+ stbir__simdf8_madd_mem( tot0, tot0, c, decode+(ofs)*3 - 1 );
5161+
5162+ #define stbir__3_coeff_remnant( ofs ) \
5163+ STBIR_SIMD_NO_UNROLL(decode); \
5164+ stbir__simdf8_load4b( cs, hc + (ofs) ); \
5165+ stbir__simdf8_0123to00001111( c, cs ); \
5166+ stbir__simdf8_madd_mem( tot0, tot0, c, decode+(ofs)*3 - 1 ); \
5167+ stbir__simdf8_0123to2222( t, cs ); \
5168+ stbir__simdf8_madd_mem4( tot1, tot1, t, decode+(ofs)*3 + 6 - 1 );
5169+
5170+#define stbir__store_output() \
5171+ stbir__simdf8_add( tot0, tot0, tot1 ); \
5172+ stbir__simdf_0123to1230( t, stbir__if_simdf8_cast_to_simdf4( tot0 ) ); \
5173+ stbir__simdf8_add4halves( t, t, tot0 ); \
5174+ horizontal_coefficients += coefficient_width; \
5175+ ++horizontal_contributors; \
5176+ output += 3; \
5177+ if ( output < output_end ) \
5178+ { \
5179+ stbir__simdf_store( output-3, t ); \
5180+ continue; \
5181+ } \
5182+ { stbir__simdf tt; stbir__simdf_0123to2301( tt, t ); \
5183+ stbir__simdf_store2( output-3, t ); \
5184+ stbir__simdf_store1( output+2-3, tt ); } \
5185+ break;
5186+
5187+
5188+#else
5189+
5190+#define stbir__4_coeff_start() \
5191+ stbir__simdf tot0,tot1,tot2,c,cs; \
5192+ STBIR_SIMD_NO_UNROLL(decode); \
5193+ stbir__simdf_load( cs, hc ); \
5194+ stbir__simdf_0123to0001( c, cs ); \
5195+ stbir__simdf_mult_mem( tot0, c, decode ); \
5196+ stbir__simdf_0123to1122( c, cs ); \
5197+ stbir__simdf_mult_mem( tot1, c, decode+4 ); \
5198+ stbir__simdf_0123to2333( c, cs ); \
5199+ stbir__simdf_mult_mem( tot2, c, decode+8 );
5200+
5201+#define stbir__4_coeff_continue_from_4( ofs ) \
5202+ STBIR_SIMD_NO_UNROLL(decode); \
5203+ stbir__simdf_load( cs, hc + (ofs) ); \
5204+ stbir__simdf_0123to0001( c, cs ); \
5205+ stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*3 ); \
5206+ stbir__simdf_0123to1122( c, cs ); \
5207+ stbir__simdf_madd_mem( tot1, tot1, c, decode+(ofs)*3+4 ); \
5208+ stbir__simdf_0123to2333( c, cs ); \
5209+ stbir__simdf_madd_mem( tot2, tot2, c, decode+(ofs)*3+8 );
5210+
5211+#define stbir__1_coeff_remnant( ofs ) \
5212+ STBIR_SIMD_NO_UNROLL(decode); \
5213+ stbir__simdf_load1z( c, hc + (ofs) ); \
5214+ stbir__simdf_0123to0001( c, c ); \
5215+ stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*3 );
5216+
5217+#define stbir__2_coeff_remnant( ofs ) \
5218+ { stbir__simdf d; \
5219+ STBIR_SIMD_NO_UNROLL(decode); \
5220+ stbir__simdf_load2z( cs, hc + (ofs) ); \
5221+ stbir__simdf_0123to0001( c, cs ); \
5222+ stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*3 ); \
5223+ stbir__simdf_0123to1122( c, cs ); \
5224+ stbir__simdf_load2z( d, decode+(ofs)*3+4 ); \
5225+ stbir__simdf_madd( tot1, tot1, c, d ); }
5226+
5227+#define stbir__3_coeff_remnant( ofs ) \
5228+ { stbir__simdf d; \
5229+ STBIR_SIMD_NO_UNROLL(decode); \
5230+ stbir__simdf_load( cs, hc + (ofs) ); \
5231+ stbir__simdf_0123to0001( c, cs ); \
5232+ stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*3 ); \
5233+ stbir__simdf_0123to1122( c, cs ); \
5234+ stbir__simdf_madd_mem( tot1, tot1, c, decode+(ofs)*3+4 ); \
5235+ stbir__simdf_0123to2222( c, cs ); \
5236+ stbir__simdf_load1z( d, decode+(ofs)*3+8 ); \
5237+ stbir__simdf_madd( tot2, tot2, c, d ); }
5238+
5239+#define stbir__store_output() \
5240+ stbir__simdf_0123ABCDto3ABx( c, tot0, tot1 ); \
5241+ stbir__simdf_0123ABCDto23Ax( cs, tot1, tot2 ); \
5242+ stbir__simdf_0123to1230( tot2, tot2 ); \
5243+ stbir__simdf_add( tot0, tot0, cs ); \
5244+ stbir__simdf_add( c, c, tot2 ); \
5245+ stbir__simdf_add( tot0, tot0, c ); \
5246+ horizontal_coefficients += coefficient_width; \
5247+ ++horizontal_contributors; \
5248+ output += 3; \
5249+ if ( output < output_end ) \
5250+ { \
5251+ stbir__simdf_store( output-3, tot0 ); \
5252+ continue; \
5253+ } \
5254+ stbir__simdf_0123to2301( tot1, tot0 ); \
5255+ stbir__simdf_store2( output-3, tot0 ); \
5256+ stbir__simdf_store1( output+2-3, tot1 ); \
5257+ break;
5258+
5259+#endif
5260+
5261+#else
5262+
5263+#define stbir__1_coeff_only() \
5264+ float tot0, tot1, tot2, c; \
5265+ c = hc[0]; \
5266+ tot0 = decode[0]*c; \
5267+ tot1 = decode[1]*c; \
5268+ tot2 = decode[2]*c;
5269+
5270+#define stbir__2_coeff_only() \
5271+ float tot0, tot1, tot2, c; \
5272+ c = hc[0]; \
5273+ tot0 = decode[0]*c; \
5274+ tot1 = decode[1]*c; \
5275+ tot2 = decode[2]*c; \
5276+ c = hc[1]; \
5277+ tot0 += decode[3]*c; \
5278+ tot1 += decode[4]*c; \
5279+ tot2 += decode[5]*c;
5280+
5281+#define stbir__3_coeff_only() \
5282+ float tot0, tot1, tot2, c; \
5283+ c = hc[0]; \
5284+ tot0 = decode[0]*c; \
5285+ tot1 = decode[1]*c; \
5286+ tot2 = decode[2]*c; \
5287+ c = hc[1]; \
5288+ tot0 += decode[3]*c; \
5289+ tot1 += decode[4]*c; \
5290+ tot2 += decode[5]*c; \
5291+ c = hc[2]; \
5292+ tot0 += decode[6]*c; \
5293+ tot1 += decode[7]*c; \
5294+ tot2 += decode[8]*c;
5295+
5296+#define stbir__store_output_tiny() \
5297+ output[0] = tot0; \
5298+ output[1] = tot1; \
5299+ output[2] = tot2; \
5300+ horizontal_coefficients += coefficient_width; \
5301+ ++horizontal_contributors; \
5302+ output += 3;
5303+
5304+#define stbir__4_coeff_start() \
5305+ float tota0,tota1,tota2,totb0,totb1,totb2,totc0,totc1,totc2,totd0,totd1,totd2,c; \
5306+ c = hc[0]; \
5307+ tota0 = decode[0]*c; \
5308+ tota1 = decode[1]*c; \
5309+ tota2 = decode[2]*c; \
5310+ c = hc[1]; \
5311+ totb0 = decode[3]*c; \
5312+ totb1 = decode[4]*c; \
5313+ totb2 = decode[5]*c; \
5314+ c = hc[2]; \
5315+ totc0 = decode[6]*c; \
5316+ totc1 = decode[7]*c; \
5317+ totc2 = decode[8]*c; \
5318+ c = hc[3]; \
5319+ totd0 = decode[9]*c; \
5320+ totd1 = decode[10]*c; \
5321+ totd2 = decode[11]*c;
5322+
5323+#define stbir__4_coeff_continue_from_4( ofs ) \
5324+ c = hc[0+(ofs)]; \
5325+ tota0 += decode[0+(ofs)*3]*c; \
5326+ tota1 += decode[1+(ofs)*3]*c; \
5327+ tota2 += decode[2+(ofs)*3]*c; \
5328+ c = hc[1+(ofs)]; \
5329+ totb0 += decode[3+(ofs)*3]*c; \
5330+ totb1 += decode[4+(ofs)*3]*c; \
5331+ totb2 += decode[5+(ofs)*3]*c; \
5332+ c = hc[2+(ofs)]; \
5333+ totc0 += decode[6+(ofs)*3]*c; \
5334+ totc1 += decode[7+(ofs)*3]*c; \
5335+ totc2 += decode[8+(ofs)*3]*c; \
5336+ c = hc[3+(ofs)]; \
5337+ totd0 += decode[9+(ofs)*3]*c; \
5338+ totd1 += decode[10+(ofs)*3]*c; \
5339+ totd2 += decode[11+(ofs)*3]*c;
5340+
5341+#define stbir__1_coeff_remnant( ofs ) \
5342+ c = hc[0+(ofs)]; \
5343+ tota0 += decode[0+(ofs)*3]*c; \
5344+ tota1 += decode[1+(ofs)*3]*c; \
5345+ tota2 += decode[2+(ofs)*3]*c;
5346+
5347+#define stbir__2_coeff_remnant( ofs ) \
5348+ c = hc[0+(ofs)]; \
5349+ tota0 += decode[0+(ofs)*3]*c; \
5350+ tota1 += decode[1+(ofs)*3]*c; \
5351+ tota2 += decode[2+(ofs)*3]*c; \
5352+ c = hc[1+(ofs)]; \
5353+ totb0 += decode[3+(ofs)*3]*c; \
5354+ totb1 += decode[4+(ofs)*3]*c; \
5355+ totb2 += decode[5+(ofs)*3]*c; \
5356+
5357+#define stbir__3_coeff_remnant( ofs ) \
5358+ c = hc[0+(ofs)]; \
5359+ tota0 += decode[0+(ofs)*3]*c; \
5360+ tota1 += decode[1+(ofs)*3]*c; \
5361+ tota2 += decode[2+(ofs)*3]*c; \
5362+ c = hc[1+(ofs)]; \
5363+ totb0 += decode[3+(ofs)*3]*c; \
5364+ totb1 += decode[4+(ofs)*3]*c; \
5365+ totb2 += decode[5+(ofs)*3]*c; \
5366+ c = hc[2+(ofs)]; \
5367+ totc0 += decode[6+(ofs)*3]*c; \
5368+ totc1 += decode[7+(ofs)*3]*c; \
5369+ totc2 += decode[8+(ofs)*3]*c;
5370+
5371+#define stbir__store_output() \
5372+ output[0] = (tota0+totc0)+(totb0+totd0); \
5373+ output[1] = (tota1+totc1)+(totb1+totd1); \
5374+ output[2] = (tota2+totc2)+(totb2+totd2); \
5375+ horizontal_coefficients += coefficient_width; \
5376+ ++horizontal_contributors; \
5377+ output += 3;
5378+
5379+#endif
5380+
5381+#define STBIR__horizontal_channels 3
5382+#define STB_IMAGE_RESIZE_DO_HORIZONTALS
5383+#include STBIR__HEADER_FILENAME
5384+
5385+//=================
5386+// Do 4 channel horizontal routines
5387+
5388+#ifdef STBIR_SIMD
5389+
5390+#define stbir__1_coeff_only() \
5391+ stbir__simdf tot,c; \
5392+ STBIR_SIMD_NO_UNROLL(decode); \
5393+ stbir__simdf_load1( c, hc ); \
5394+ stbir__simdf_0123to0000( c, c ); \
5395+ stbir__simdf_mult_mem( tot, c, decode );
5396+
5397+#define stbir__2_coeff_only() \
5398+ stbir__simdf tot,c,cs; \
5399+ STBIR_SIMD_NO_UNROLL(decode); \
5400+ stbir__simdf_load2( cs, hc ); \
5401+ stbir__simdf_0123to0000( c, cs ); \
5402+ stbir__simdf_mult_mem( tot, c, decode ); \
5403+ stbir__simdf_0123to1111( c, cs ); \
5404+ stbir__simdf_madd_mem( tot, tot, c, decode+4 );
5405+
5406+#define stbir__3_coeff_only() \
5407+ stbir__simdf tot,c,cs; \
5408+ STBIR_SIMD_NO_UNROLL(decode); \
5409+ stbir__simdf_load( cs, hc ); \
5410+ stbir__simdf_0123to0000( c, cs ); \
5411+ stbir__simdf_mult_mem( tot, c, decode ); \
5412+ stbir__simdf_0123to1111( c, cs ); \
5413+ stbir__simdf_madd_mem( tot, tot, c, decode+4 ); \
5414+ stbir__simdf_0123to2222( c, cs ); \
5415+ stbir__simdf_madd_mem( tot, tot, c, decode+8 );
5416+
5417+#define stbir__store_output_tiny() \
5418+ stbir__simdf_store( output, tot ); \
5419+ horizontal_coefficients += coefficient_width; \
5420+ ++horizontal_contributors; \
5421+ output += 4;
5422+
5423+#ifdef STBIR_SIMD8
5424+
5425+#define stbir__4_coeff_start() \
5426+ stbir__simdf8 tot0,c,cs; stbir__simdf t; \
5427+ STBIR_SIMD_NO_UNROLL(decode); \
5428+ stbir__simdf8_load4b( cs, hc ); \
5429+ stbir__simdf8_0123to00001111( c, cs ); \
5430+ stbir__simdf8_mult_mem( tot0, c, decode ); \
5431+ stbir__simdf8_0123to22223333( c, cs ); \
5432+ stbir__simdf8_madd_mem( tot0, tot0, c, decode+8 );
5433+
5434+#define stbir__4_coeff_continue_from_4( ofs ) \
5435+ STBIR_SIMD_NO_UNROLL(decode); \
5436+ stbir__simdf8_load4b( cs, hc + (ofs) ); \
5437+ stbir__simdf8_0123to00001111( c, cs ); \
5438+ stbir__simdf8_madd_mem( tot0, tot0, c, decode+(ofs)*4 ); \
5439+ stbir__simdf8_0123to22223333( c, cs ); \
5440+ stbir__simdf8_madd_mem( tot0, tot0, c, decode+(ofs)*4+8 );
5441+
5442+#define stbir__1_coeff_remnant( ofs ) \
5443+ STBIR_SIMD_NO_UNROLL(decode); \
5444+ stbir__simdf_load1rep4( t, hc + (ofs) ); \
5445+ stbir__simdf8_madd_mem4( tot0, tot0, t, decode+(ofs)*4 );
5446+
5447+#define stbir__2_coeff_remnant( ofs ) \
5448+ STBIR_SIMD_NO_UNROLL(decode); \
5449+ stbir__simdf8_load4b( cs, hc + (ofs) - 2 ); \
5450+ stbir__simdf8_0123to22223333( c, cs ); \
5451+ stbir__simdf8_madd_mem( tot0, tot0, c, decode+(ofs)*4 );
5452+
5453+ #define stbir__3_coeff_remnant( ofs ) \
5454+ STBIR_SIMD_NO_UNROLL(decode); \
5455+ stbir__simdf8_load4b( cs, hc + (ofs) ); \
5456+ stbir__simdf8_0123to00001111( c, cs ); \
5457+ stbir__simdf8_madd_mem( tot0, tot0, c, decode+(ofs)*4 ); \
5458+ stbir__simdf8_0123to2222( t, cs ); \
5459+ stbir__simdf8_madd_mem4( tot0, tot0, t, decode+(ofs)*4+8 );
5460+
5461+#define stbir__store_output() \
5462+ stbir__simdf8_add4halves( t, stbir__if_simdf8_cast_to_simdf4(tot0), tot0 ); \
5463+ stbir__simdf_store( output, t ); \
5464+ horizontal_coefficients += coefficient_width; \
5465+ ++horizontal_contributors; \
5466+ output += 4;
5467+
5468+#else
5469+
5470+#define stbir__4_coeff_start() \
5471+ stbir__simdf tot0,tot1,c,cs; \
5472+ STBIR_SIMD_NO_UNROLL(decode); \
5473+ stbir__simdf_load( cs, hc ); \
5474+ stbir__simdf_0123to0000( c, cs ); \
5475+ stbir__simdf_mult_mem( tot0, c, decode ); \
5476+ stbir__simdf_0123to1111( c, cs ); \
5477+ stbir__simdf_mult_mem( tot1, c, decode+4 ); \
5478+ stbir__simdf_0123to2222( c, cs ); \
5479+ stbir__simdf_madd_mem( tot0, tot0, c, decode+8 ); \
5480+ stbir__simdf_0123to3333( c, cs ); \
5481+ stbir__simdf_madd_mem( tot1, tot1, c, decode+12 );
5482+
5483+#define stbir__4_coeff_continue_from_4( ofs ) \
5484+ STBIR_SIMD_NO_UNROLL(decode); \
5485+ stbir__simdf_load( cs, hc + (ofs) ); \
5486+ stbir__simdf_0123to0000( c, cs ); \
5487+ stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*4 ); \
5488+ stbir__simdf_0123to1111( c, cs ); \
5489+ stbir__simdf_madd_mem( tot1, tot1, c, decode+(ofs)*4+4 ); \
5490+ stbir__simdf_0123to2222( c, cs ); \
5491+ stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*4+8 ); \
5492+ stbir__simdf_0123to3333( c, cs ); \
5493+ stbir__simdf_madd_mem( tot1, tot1, c, decode+(ofs)*4+12 );
5494+
5495+#define stbir__1_coeff_remnant( ofs ) \
5496+ STBIR_SIMD_NO_UNROLL(decode); \
5497+ stbir__simdf_load1( c, hc + (ofs) ); \
5498+ stbir__simdf_0123to0000( c, c ); \
5499+ stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*4 );
5500+
5501+#define stbir__2_coeff_remnant( ofs ) \
5502+ STBIR_SIMD_NO_UNROLL(decode); \
5503+ stbir__simdf_load2( cs, hc + (ofs) ); \
5504+ stbir__simdf_0123to0000( c, cs ); \
5505+ stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*4 ); \
5506+ stbir__simdf_0123to1111( c, cs ); \
5507+ stbir__simdf_madd_mem( tot1, tot1, c, decode+(ofs)*4+4 );
5508+
5509+#define stbir__3_coeff_remnant( ofs ) \
5510+ STBIR_SIMD_NO_UNROLL(decode); \
5511+ stbir__simdf_load( cs, hc + (ofs) ); \
5512+ stbir__simdf_0123to0000( c, cs ); \
5513+ stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*4 ); \
5514+ stbir__simdf_0123to1111( c, cs ); \
5515+ stbir__simdf_madd_mem( tot1, tot1, c, decode+(ofs)*4+4 ); \
5516+ stbir__simdf_0123to2222( c, cs ); \
5517+ stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*4+8 );
5518+
5519+#define stbir__store_output() \
5520+ stbir__simdf_add( tot0, tot0, tot1 ); \
5521+ stbir__simdf_store( output, tot0 ); \
5522+ horizontal_coefficients += coefficient_width; \
5523+ ++horizontal_contributors; \
5524+ output += 4;
5525+
5526+#endif
5527+
5528+#else
5529+
5530+#define stbir__1_coeff_only() \
5531+ float p0,p1,p2,p3,c; \
5532+ STBIR_SIMD_NO_UNROLL(decode); \
5533+ c = hc[0]; \
5534+ p0 = decode[0] * c; \
5535+ p1 = decode[1] * c; \
5536+ p2 = decode[2] * c; \
5537+ p3 = decode[3] * c;
5538+
5539+#define stbir__2_coeff_only() \
5540+ float p0,p1,p2,p3,c; \
5541+ STBIR_SIMD_NO_UNROLL(decode); \
5542+ c = hc[0]; \
5543+ p0 = decode[0] * c; \
5544+ p1 = decode[1] * c; \
5545+ p2 = decode[2] * c; \
5546+ p3 = decode[3] * c; \
5547+ c = hc[1]; \
5548+ p0 += decode[4] * c; \
5549+ p1 += decode[5] * c; \
5550+ p2 += decode[6] * c; \
5551+ p3 += decode[7] * c;
5552+
5553+#define stbir__3_coeff_only() \
5554+ float p0,p1,p2,p3,c; \
5555+ STBIR_SIMD_NO_UNROLL(decode); \
5556+ c = hc[0]; \
5557+ p0 = decode[0] * c; \
5558+ p1 = decode[1] * c; \
5559+ p2 = decode[2] * c; \
5560+ p3 = decode[3] * c; \
5561+ c = hc[1]; \
5562+ p0 += decode[4] * c; \
5563+ p1 += decode[5] * c; \
5564+ p2 += decode[6] * c; \
5565+ p3 += decode[7] * c; \
5566+ c = hc[2]; \
5567+ p0 += decode[8] * c; \
5568+ p1 += decode[9] * c; \
5569+ p2 += decode[10] * c; \
5570+ p3 += decode[11] * c;
5571+
5572+#define stbir__store_output_tiny() \
5573+ output[0] = p0; \
5574+ output[1] = p1; \
5575+ output[2] = p2; \
5576+ output[3] = p3; \
5577+ horizontal_coefficients += coefficient_width; \
5578+ ++horizontal_contributors; \
5579+ output += 4;
5580+
5581+#define stbir__4_coeff_start() \
5582+ float x0,x1,x2,x3,y0,y1,y2,y3,c; \
5583+ STBIR_SIMD_NO_UNROLL(decode); \
5584+ c = hc[0]; \
5585+ x0 = decode[0] * c; \
5586+ x1 = decode[1] * c; \
5587+ x2 = decode[2] * c; \
5588+ x3 = decode[3] * c; \
5589+ c = hc[1]; \
5590+ y0 = decode[4] * c; \
5591+ y1 = decode[5] * c; \
5592+ y2 = decode[6] * c; \
5593+ y3 = decode[7] * c; \
5594+ c = hc[2]; \
5595+ x0 += decode[8] * c; \
5596+ x1 += decode[9] * c; \
5597+ x2 += decode[10] * c; \
5598+ x3 += decode[11] * c; \
5599+ c = hc[3]; \
5600+ y0 += decode[12] * c; \
5601+ y1 += decode[13] * c; \
5602+ y2 += decode[14] * c; \
5603+ y3 += decode[15] * c;
5604+
5605+#define stbir__4_coeff_continue_from_4( ofs ) \
5606+ STBIR_SIMD_NO_UNROLL(decode); \
5607+ c = hc[0+(ofs)]; \
5608+ x0 += decode[0+(ofs)*4] * c; \
5609+ x1 += decode[1+(ofs)*4] * c; \
5610+ x2 += decode[2+(ofs)*4] * c; \
5611+ x3 += decode[3+(ofs)*4] * c; \
5612+ c = hc[1+(ofs)]; \
5613+ y0 += decode[4+(ofs)*4] * c; \
5614+ y1 += decode[5+(ofs)*4] * c; \
5615+ y2 += decode[6+(ofs)*4] * c; \
5616+ y3 += decode[7+(ofs)*4] * c; \
5617+ c = hc[2+(ofs)]; \
5618+ x0 += decode[8+(ofs)*4] * c; \
5619+ x1 += decode[9+(ofs)*4] * c; \
5620+ x2 += decode[10+(ofs)*4] * c; \
5621+ x3 += decode[11+(ofs)*4] * c; \
5622+ c = hc[3+(ofs)]; \
5623+ y0 += decode[12+(ofs)*4] * c; \
5624+ y1 += decode[13+(ofs)*4] * c; \
5625+ y2 += decode[14+(ofs)*4] * c; \
5626+ y3 += decode[15+(ofs)*4] * c;
5627+
5628+#define stbir__1_coeff_remnant( ofs ) \
5629+ STBIR_SIMD_NO_UNROLL(decode); \
5630+ c = hc[0+(ofs)]; \
5631+ x0 += decode[0+(ofs)*4] * c; \
5632+ x1 += decode[1+(ofs)*4] * c; \
5633+ x2 += decode[2+(ofs)*4] * c; \
5634+ x3 += decode[3+(ofs)*4] * c;
5635+
5636+#define stbir__2_coeff_remnant( ofs ) \
5637+ STBIR_SIMD_NO_UNROLL(decode); \
5638+ c = hc[0+(ofs)]; \
5639+ x0 += decode[0+(ofs)*4] * c; \
5640+ x1 += decode[1+(ofs)*4] * c; \
5641+ x2 += decode[2+(ofs)*4] * c; \
5642+ x3 += decode[3+(ofs)*4] * c; \
5643+ c = hc[1+(ofs)]; \
5644+ y0 += decode[4+(ofs)*4] * c; \
5645+ y1 += decode[5+(ofs)*4] * c; \
5646+ y2 += decode[6+(ofs)*4] * c; \
5647+ y3 += decode[7+(ofs)*4] * c;
5648+
5649+#define stbir__3_coeff_remnant( ofs ) \
5650+ STBIR_SIMD_NO_UNROLL(decode); \
5651+ c = hc[0+(ofs)]; \
5652+ x0 += decode[0+(ofs)*4] * c; \
5653+ x1 += decode[1+(ofs)*4] * c; \
5654+ x2 += decode[2+(ofs)*4] * c; \
5655+ x3 += decode[3+(ofs)*4] * c; \
5656+ c = hc[1+(ofs)]; \
5657+ y0 += decode[4+(ofs)*4] * c; \
5658+ y1 += decode[5+(ofs)*4] * c; \
5659+ y2 += decode[6+(ofs)*4] * c; \
5660+ y3 += decode[7+(ofs)*4] * c; \
5661+ c = hc[2+(ofs)]; \
5662+ x0 += decode[8+(ofs)*4] * c; \
5663+ x1 += decode[9+(ofs)*4] * c; \
5664+ x2 += decode[10+(ofs)*4] * c; \
5665+ x3 += decode[11+(ofs)*4] * c;
5666+
5667+#define stbir__store_output() \
5668+ output[0] = x0 + y0; \
5669+ output[1] = x1 + y1; \
5670+ output[2] = x2 + y2; \
5671+ output[3] = x3 + y3; \
5672+ horizontal_coefficients += coefficient_width; \
5673+ ++horizontal_contributors; \
5674+ output += 4;
5675+
5676+#endif
5677+
5678+#define STBIR__horizontal_channels 4
5679+#define STB_IMAGE_RESIZE_DO_HORIZONTALS
5680+#include STBIR__HEADER_FILENAME
5681+
5682+
5683+
5684+//=================
5685+// Do 7 channel horizontal routines
5686+
5687+#ifdef STBIR_SIMD
5688+
5689+#define stbir__1_coeff_only() \
5690+ stbir__simdf tot0,tot1,c; \
5691+ STBIR_SIMD_NO_UNROLL(decode); \
5692+ stbir__simdf_load1( c, hc ); \
5693+ stbir__simdf_0123to0000( c, c ); \
5694+ stbir__simdf_mult_mem( tot0, c, decode ); \
5695+ stbir__simdf_mult_mem( tot1, c, decode+3 );
5696+
5697+#define stbir__2_coeff_only() \
5698+ stbir__simdf tot0,tot1,c,cs; \
5699+ STBIR_SIMD_NO_UNROLL(decode); \
5700+ stbir__simdf_load2( cs, hc ); \
5701+ stbir__simdf_0123to0000( c, cs ); \
5702+ stbir__simdf_mult_mem( tot0, c, decode ); \
5703+ stbir__simdf_mult_mem( tot1, c, decode+3 ); \
5704+ stbir__simdf_0123to1111( c, cs ); \
5705+ stbir__simdf_madd_mem( tot0, tot0, c, decode+7 ); \
5706+ stbir__simdf_madd_mem( tot1, tot1, c,decode+10 );
5707+
5708+#define stbir__3_coeff_only() \
5709+ stbir__simdf tot0,tot1,c,cs; \
5710+ STBIR_SIMD_NO_UNROLL(decode); \
5711+ stbir__simdf_load( cs, hc ); \
5712+ stbir__simdf_0123to0000( c, cs ); \
5713+ stbir__simdf_mult_mem( tot0, c, decode ); \
5714+ stbir__simdf_mult_mem( tot1, c, decode+3 ); \
5715+ stbir__simdf_0123to1111( c, cs ); \
5716+ stbir__simdf_madd_mem( tot0, tot0, c, decode+7 ); \
5717+ stbir__simdf_madd_mem( tot1, tot1, c, decode+10 ); \
5718+ stbir__simdf_0123to2222( c, cs ); \
5719+ stbir__simdf_madd_mem( tot0, tot0, c, decode+14 ); \
5720+ stbir__simdf_madd_mem( tot1, tot1, c, decode+17 );
5721+
5722+#define stbir__store_output_tiny() \
5723+ stbir__simdf_store( output+3, tot1 ); \
5724+ stbir__simdf_store( output, tot0 ); \
5725+ horizontal_coefficients += coefficient_width; \
5726+ ++horizontal_contributors; \
5727+ output += 7;
5728+
5729+#ifdef STBIR_SIMD8
5730+
5731+#define stbir__4_coeff_start() \
5732+ stbir__simdf8 tot0,tot1,c,cs; \
5733+ STBIR_SIMD_NO_UNROLL(decode); \
5734+ stbir__simdf8_load4b( cs, hc ); \
5735+ stbir__simdf8_0123to00000000( c, cs ); \
5736+ stbir__simdf8_mult_mem( tot0, c, decode ); \
5737+ stbir__simdf8_0123to11111111( c, cs ); \
5738+ stbir__simdf8_mult_mem( tot1, c, decode+7 ); \
5739+ stbir__simdf8_0123to22222222( c, cs ); \
5740+ stbir__simdf8_madd_mem( tot0, tot0, c, decode+14 ); \
5741+ stbir__simdf8_0123to33333333( c, cs ); \
5742+ stbir__simdf8_madd_mem( tot1, tot1, c, decode+21 );
5743+
5744+#define stbir__4_coeff_continue_from_4( ofs ) \
5745+ STBIR_SIMD_NO_UNROLL(decode); \
5746+ stbir__simdf8_load4b( cs, hc + (ofs) ); \
5747+ stbir__simdf8_0123to00000000( c, cs ); \
5748+ stbir__simdf8_madd_mem( tot0, tot0, c, decode+(ofs)*7 ); \
5749+ stbir__simdf8_0123to11111111( c, cs ); \
5750+ stbir__simdf8_madd_mem( tot1, tot1, c, decode+(ofs)*7+7 ); \
5751+ stbir__simdf8_0123to22222222( c, cs ); \
5752+ stbir__simdf8_madd_mem( tot0, tot0, c, decode+(ofs)*7+14 ); \
5753+ stbir__simdf8_0123to33333333( c, cs ); \
5754+ stbir__simdf8_madd_mem( tot1, tot1, c, decode+(ofs)*7+21 );
5755+
5756+#define stbir__1_coeff_remnant( ofs ) \
5757+ STBIR_SIMD_NO_UNROLL(decode); \
5758+ stbir__simdf8_load1b( c, hc + (ofs) ); \
5759+ stbir__simdf8_madd_mem( tot0, tot0, c, decode+(ofs)*7 );
5760+
5761+#define stbir__2_coeff_remnant( ofs ) \
5762+ STBIR_SIMD_NO_UNROLL(decode); \
5763+ stbir__simdf8_load1b( c, hc + (ofs) ); \
5764+ stbir__simdf8_madd_mem( tot0, tot0, c, decode+(ofs)*7 ); \
5765+ stbir__simdf8_load1b( c, hc + (ofs)+1 ); \
5766+ stbir__simdf8_madd_mem( tot1, tot1, c, decode+(ofs)*7+7 );
5767+
5768+#define stbir__3_coeff_remnant( ofs ) \
5769+ STBIR_SIMD_NO_UNROLL(decode); \
5770+ stbir__simdf8_load4b( cs, hc + (ofs) ); \
5771+ stbir__simdf8_0123to00000000( c, cs ); \
5772+ stbir__simdf8_madd_mem( tot0, tot0, c, decode+(ofs)*7 ); \
5773+ stbir__simdf8_0123to11111111( c, cs ); \
5774+ stbir__simdf8_madd_mem( tot1, tot1, c, decode+(ofs)*7+7 ); \
5775+ stbir__simdf8_0123to22222222( c, cs ); \
5776+ stbir__simdf8_madd_mem( tot0, tot0, c, decode+(ofs)*7+14 );
5777+
5778+#define stbir__store_output() \
5779+ stbir__simdf8_add( tot0, tot0, tot1 ); \
5780+ horizontal_coefficients += coefficient_width; \
5781+ ++horizontal_contributors; \
5782+ output += 7; \
5783+ if ( output < output_end ) \
5784+ { \
5785+ stbir__simdf8_store( output-7, tot0 ); \
5786+ continue; \
5787+ } \
5788+ stbir__simdf_store( output-7+3, stbir__simdf_swiz(stbir__simdf8_gettop4(tot0),0,0,1,2) ); \
5789+ stbir__simdf_store( output-7, stbir__if_simdf8_cast_to_simdf4(tot0) ); \
5790+ break;
5791+
5792+#else
5793+
5794+#define stbir__4_coeff_start() \
5795+ stbir__simdf tot0,tot1,tot2,tot3,c,cs; \
5796+ STBIR_SIMD_NO_UNROLL(decode); \
5797+ stbir__simdf_load( cs, hc ); \
5798+ stbir__simdf_0123to0000( c, cs ); \
5799+ stbir__simdf_mult_mem( tot0, c, decode ); \
5800+ stbir__simdf_mult_mem( tot1, c, decode+3 ); \
5801+ stbir__simdf_0123to1111( c, cs ); \
5802+ stbir__simdf_mult_mem( tot2, c, decode+7 ); \
5803+ stbir__simdf_mult_mem( tot3, c, decode+10 ); \
5804+ stbir__simdf_0123to2222( c, cs ); \
5805+ stbir__simdf_madd_mem( tot0, tot0, c, decode+14 ); \
5806+ stbir__simdf_madd_mem( tot1, tot1, c, decode+17 ); \
5807+ stbir__simdf_0123to3333( c, cs ); \
5808+ stbir__simdf_madd_mem( tot2, tot2, c, decode+21 ); \
5809+ stbir__simdf_madd_mem( tot3, tot3, c, decode+24 );
5810+
5811+#define stbir__4_coeff_continue_from_4( ofs ) \
5812+ STBIR_SIMD_NO_UNROLL(decode); \
5813+ stbir__simdf_load( cs, hc + (ofs) ); \
5814+ stbir__simdf_0123to0000( c, cs ); \
5815+ stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*7 ); \
5816+ stbir__simdf_madd_mem( tot1, tot1, c, decode+(ofs)*7+3 ); \
5817+ stbir__simdf_0123to1111( c, cs ); \
5818+ stbir__simdf_madd_mem( tot2, tot2, c, decode+(ofs)*7+7 ); \
5819+ stbir__simdf_madd_mem( tot3, tot3, c, decode+(ofs)*7+10 ); \
5820+ stbir__simdf_0123to2222( c, cs ); \
5821+ stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*7+14 ); \
5822+ stbir__simdf_madd_mem( tot1, tot1, c, decode+(ofs)*7+17 ); \
5823+ stbir__simdf_0123to3333( c, cs ); \
5824+ stbir__simdf_madd_mem( tot2, tot2, c, decode+(ofs)*7+21 ); \
5825+ stbir__simdf_madd_mem( tot3, tot3, c, decode+(ofs)*7+24 );
5826+
5827+#define stbir__1_coeff_remnant( ofs ) \
5828+ STBIR_SIMD_NO_UNROLL(decode); \
5829+ stbir__simdf_load1( c, hc + (ofs) ); \
5830+ stbir__simdf_0123to0000( c, c ); \
5831+ stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*7 ); \
5832+ stbir__simdf_madd_mem( tot1, tot1, c, decode+(ofs)*7+3 ); \
5833+
5834+#define stbir__2_coeff_remnant( ofs ) \
5835+ STBIR_SIMD_NO_UNROLL(decode); \
5836+ stbir__simdf_load2( cs, hc + (ofs) ); \
5837+ stbir__simdf_0123to0000( c, cs ); \
5838+ stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*7 ); \
5839+ stbir__simdf_madd_mem( tot1, tot1, c, decode+(ofs)*7+3 ); \
5840+ stbir__simdf_0123to1111( c, cs ); \
5841+ stbir__simdf_madd_mem( tot2, tot2, c, decode+(ofs)*7+7 ); \
5842+ stbir__simdf_madd_mem( tot3, tot3, c, decode+(ofs)*7+10 );
5843+
5844+#define stbir__3_coeff_remnant( ofs ) \
5845+ STBIR_SIMD_NO_UNROLL(decode); \
5846+ stbir__simdf_load( cs, hc + (ofs) ); \
5847+ stbir__simdf_0123to0000( c, cs ); \
5848+ stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*7 ); \
5849+ stbir__simdf_madd_mem( tot1, tot1, c, decode+(ofs)*7+3 ); \
5850+ stbir__simdf_0123to1111( c, cs ); \
5851+ stbir__simdf_madd_mem( tot2, tot2, c, decode+(ofs)*7+7 ); \
5852+ stbir__simdf_madd_mem( tot3, tot3, c, decode+(ofs)*7+10 ); \
5853+ stbir__simdf_0123to2222( c, cs ); \
5854+ stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*7+14 ); \
5855+ stbir__simdf_madd_mem( tot1, tot1, c, decode+(ofs)*7+17 );
5856+
5857+#define stbir__store_output() \
5858+ stbir__simdf_add( tot0, tot0, tot2 ); \
5859+ stbir__simdf_add( tot1, tot1, tot3 ); \
5860+ stbir__simdf_store( output+3, tot1 ); \
5861+ stbir__simdf_store( output, tot0 ); \
5862+ horizontal_coefficients += coefficient_width; \
5863+ ++horizontal_contributors; \
5864+ output += 7;
5865+
5866+#endif
5867+
5868+#else
5869+
5870+#define stbir__1_coeff_only() \
5871+ float tot0, tot1, tot2, tot3, tot4, tot5, tot6, c; \
5872+ c = hc[0]; \
5873+ tot0 = decode[0]*c; \
5874+ tot1 = decode[1]*c; \
5875+ tot2 = decode[2]*c; \
5876+ tot3 = decode[3]*c; \
5877+ tot4 = decode[4]*c; \
5878+ tot5 = decode[5]*c; \
5879+ tot6 = decode[6]*c;
5880+
5881+#define stbir__2_coeff_only() \
5882+ float tot0, tot1, tot2, tot3, tot4, tot5, tot6, c; \
5883+ c = hc[0]; \
5884+ tot0 = decode[0]*c; \
5885+ tot1 = decode[1]*c; \
5886+ tot2 = decode[2]*c; \
5887+ tot3 = decode[3]*c; \
5888+ tot4 = decode[4]*c; \
5889+ tot5 = decode[5]*c; \
5890+ tot6 = decode[6]*c; \
5891+ c = hc[1]; \
5892+ tot0 += decode[7]*c; \
5893+ tot1 += decode[8]*c; \
5894+ tot2 += decode[9]*c; \
5895+ tot3 += decode[10]*c; \
5896+ tot4 += decode[11]*c; \
5897+ tot5 += decode[12]*c; \
5898+ tot6 += decode[13]*c; \
5899+
5900+#define stbir__3_coeff_only() \
5901+ float tot0, tot1, tot2, tot3, tot4, tot5, tot6, c; \
5902+ c = hc[0]; \
5903+ tot0 = decode[0]*c; \
5904+ tot1 = decode[1]*c; \
5905+ tot2 = decode[2]*c; \
5906+ tot3 = decode[3]*c; \
5907+ tot4 = decode[4]*c; \
5908+ tot5 = decode[5]*c; \
5909+ tot6 = decode[6]*c; \
5910+ c = hc[1]; \
5911+ tot0 += decode[7]*c; \
5912+ tot1 += decode[8]*c; \
5913+ tot2 += decode[9]*c; \
5914+ tot3 += decode[10]*c; \
5915+ tot4 += decode[11]*c; \
5916+ tot5 += decode[12]*c; \
5917+ tot6 += decode[13]*c; \
5918+ c = hc[2]; \
5919+ tot0 += decode[14]*c; \
5920+ tot1 += decode[15]*c; \
5921+ tot2 += decode[16]*c; \
5922+ tot3 += decode[17]*c; \
5923+ tot4 += decode[18]*c; \
5924+ tot5 += decode[19]*c; \
5925+ tot6 += decode[20]*c; \
5926+
5927+#define stbir__store_output_tiny() \
5928+ output[0] = tot0; \
5929+ output[1] = tot1; \
5930+ output[2] = tot2; \
5931+ output[3] = tot3; \
5932+ output[4] = tot4; \
5933+ output[5] = tot5; \
5934+ output[6] = tot6; \
5935+ horizontal_coefficients += coefficient_width; \
5936+ ++horizontal_contributors; \
5937+ output += 7;
5938+
5939+#define stbir__4_coeff_start() \
5940+ float x0,x1,x2,x3,x4,x5,x6,y0,y1,y2,y3,y4,y5,y6,c; \
5941+ STBIR_SIMD_NO_UNROLL(decode); \
5942+ c = hc[0]; \
5943+ x0 = decode[0] * c; \
5944+ x1 = decode[1] * c; \
5945+ x2 = decode[2] * c; \
5946+ x3 = decode[3] * c; \
5947+ x4 = decode[4] * c; \
5948+ x5 = decode[5] * c; \
5949+ x6 = decode[6] * c; \
5950+ c = hc[1]; \
5951+ y0 = decode[7] * c; \
5952+ y1 = decode[8] * c; \
5953+ y2 = decode[9] * c; \
5954+ y3 = decode[10] * c; \
5955+ y4 = decode[11] * c; \
5956+ y5 = decode[12] * c; \
5957+ y6 = decode[13] * c; \
5958+ c = hc[2]; \
5959+ x0 += decode[14] * c; \
5960+ x1 += decode[15] * c; \
5961+ x2 += decode[16] * c; \
5962+ x3 += decode[17] * c; \
5963+ x4 += decode[18] * c; \
5964+ x5 += decode[19] * c; \
5965+ x6 += decode[20] * c; \
5966+ c = hc[3]; \
5967+ y0 += decode[21] * c; \
5968+ y1 += decode[22] * c; \
5969+ y2 += decode[23] * c; \
5970+ y3 += decode[24] * c; \
5971+ y4 += decode[25] * c; \
5972+ y5 += decode[26] * c; \
5973+ y6 += decode[27] * c;
5974+
5975+#define stbir__4_coeff_continue_from_4( ofs ) \
5976+ STBIR_SIMD_NO_UNROLL(decode); \
5977+ c = hc[0+(ofs)]; \
5978+ x0 += decode[0+(ofs)*7] * c; \
5979+ x1 += decode[1+(ofs)*7] * c; \
5980+ x2 += decode[2+(ofs)*7] * c; \
5981+ x3 += decode[3+(ofs)*7] * c; \
5982+ x4 += decode[4+(ofs)*7] * c; \
5983+ x5 += decode[5+(ofs)*7] * c; \
5984+ x6 += decode[6+(ofs)*7] * c; \
5985+ c = hc[1+(ofs)]; \
5986+ y0 += decode[7+(ofs)*7] * c; \
5987+ y1 += decode[8+(ofs)*7] * c; \
5988+ y2 += decode[9+(ofs)*7] * c; \
5989+ y3 += decode[10+(ofs)*7] * c; \
5990+ y4 += decode[11+(ofs)*7] * c; \
5991+ y5 += decode[12+(ofs)*7] * c; \
5992+ y6 += decode[13+(ofs)*7] * c; \
5993+ c = hc[2+(ofs)]; \
5994+ x0 += decode[14+(ofs)*7] * c; \
5995+ x1 += decode[15+(ofs)*7] * c; \
5996+ x2 += decode[16+(ofs)*7] * c; \
5997+ x3 += decode[17+(ofs)*7] * c; \
5998+ x4 += decode[18+(ofs)*7] * c; \
5999+ x5 += decode[19+(ofs)*7] * c; \
6000+ x6 += decode[20+(ofs)*7] * c; \
6001+ c = hc[3+(ofs)]; \
6002+ y0 += decode[21+(ofs)*7] * c; \
6003+ y1 += decode[22+(ofs)*7] * c; \
6004+ y2 += decode[23+(ofs)*7] * c; \
6005+ y3 += decode[24+(ofs)*7] * c; \
6006+ y4 += decode[25+(ofs)*7] * c; \
6007+ y5 += decode[26+(ofs)*7] * c; \
6008+ y6 += decode[27+(ofs)*7] * c;
6009+
6010+#define stbir__1_coeff_remnant( ofs ) \
6011+ STBIR_SIMD_NO_UNROLL(decode); \
6012+ c = hc[0+(ofs)]; \
6013+ x0 += decode[0+(ofs)*7] * c; \
6014+ x1 += decode[1+(ofs)*7] * c; \
6015+ x2 += decode[2+(ofs)*7] * c; \
6016+ x3 += decode[3+(ofs)*7] * c; \
6017+ x4 += decode[4+(ofs)*7] * c; \
6018+ x5 += decode[5+(ofs)*7] * c; \
6019+ x6 += decode[6+(ofs)*7] * c; \
6020+
6021+#define stbir__2_coeff_remnant( ofs ) \
6022+ STBIR_SIMD_NO_UNROLL(decode); \
6023+ c = hc[0+(ofs)]; \
6024+ x0 += decode[0+(ofs)*7] * c; \
6025+ x1 += decode[1+(ofs)*7] * c; \
6026+ x2 += decode[2+(ofs)*7] * c; \
6027+ x3 += decode[3+(ofs)*7] * c; \
6028+ x4 += decode[4+(ofs)*7] * c; \
6029+ x5 += decode[5+(ofs)*7] * c; \
6030+ x6 += decode[6+(ofs)*7] * c; \
6031+ c = hc[1+(ofs)]; \
6032+ y0 += decode[7+(ofs)*7] * c; \
6033+ y1 += decode[8+(ofs)*7] * c; \
6034+ y2 += decode[9+(ofs)*7] * c; \
6035+ y3 += decode[10+(ofs)*7] * c; \
6036+ y4 += decode[11+(ofs)*7] * c; \
6037+ y5 += decode[12+(ofs)*7] * c; \
6038+ y6 += decode[13+(ofs)*7] * c; \
6039+
6040+#define stbir__3_coeff_remnant( ofs ) \
6041+ STBIR_SIMD_NO_UNROLL(decode); \
6042+ c = hc[0+(ofs)]; \
6043+ x0 += decode[0+(ofs)*7] * c; \
6044+ x1 += decode[1+(ofs)*7] * c; \
6045+ x2 += decode[2+(ofs)*7] * c; \
6046+ x3 += decode[3+(ofs)*7] * c; \
6047+ x4 += decode[4+(ofs)*7] * c; \
6048+ x5 += decode[5+(ofs)*7] * c; \
6049+ x6 += decode[6+(ofs)*7] * c; \
6050+ c = hc[1+(ofs)]; \
6051+ y0 += decode[7+(ofs)*7] * c; \
6052+ y1 += decode[8+(ofs)*7] * c; \
6053+ y2 += decode[9+(ofs)*7] * c; \
6054+ y3 += decode[10+(ofs)*7] * c; \
6055+ y4 += decode[11+(ofs)*7] * c; \
6056+ y5 += decode[12+(ofs)*7] * c; \
6057+ y6 += decode[13+(ofs)*7] * c; \
6058+ c = hc[2+(ofs)]; \
6059+ x0 += decode[14+(ofs)*7] * c; \
6060+ x1 += decode[15+(ofs)*7] * c; \
6061+ x2 += decode[16+(ofs)*7] * c; \
6062+ x3 += decode[17+(ofs)*7] * c; \
6063+ x4 += decode[18+(ofs)*7] * c; \
6064+ x5 += decode[19+(ofs)*7] * c; \
6065+ x6 += decode[20+(ofs)*7] * c; \
6066+
6067+#define stbir__store_output() \
6068+ output[0] = x0 + y0; \
6069+ output[1] = x1 + y1; \
6070+ output[2] = x2 + y2; \
6071+ output[3] = x3 + y3; \
6072+ output[4] = x4 + y4; \
6073+ output[5] = x5 + y5; \
6074+ output[6] = x6 + y6; \
6075+ horizontal_coefficients += coefficient_width; \
6076+ ++horizontal_contributors; \
6077+ output += 7;
6078+
6079+#endif
6080+
6081+#define STBIR__horizontal_channels 7
6082+#define STB_IMAGE_RESIZE_DO_HORIZONTALS
6083+#include STBIR__HEADER_FILENAME
6084+
6085+
6086+// include all of the vertical resamplers (both scatter and gather versions)
6087+
6088+#define STBIR__vertical_channels 1
6089+#define STB_IMAGE_RESIZE_DO_VERTICALS
6090+#include STBIR__HEADER_FILENAME
6091+
6092+#define STBIR__vertical_channels 1
6093+#define STB_IMAGE_RESIZE_DO_VERTICALS
6094+#define STB_IMAGE_RESIZE_VERTICAL_CONTINUE
6095+#include STBIR__HEADER_FILENAME
6096+
6097+#define STBIR__vertical_channels 2
6098+#define STB_IMAGE_RESIZE_DO_VERTICALS
6099+#include STBIR__HEADER_FILENAME
6100+
6101+#define STBIR__vertical_channels 2
6102+#define STB_IMAGE_RESIZE_DO_VERTICALS
6103+#define STB_IMAGE_RESIZE_VERTICAL_CONTINUE
6104+#include STBIR__HEADER_FILENAME
6105+
6106+#define STBIR__vertical_channels 3
6107+#define STB_IMAGE_RESIZE_DO_VERTICALS
6108+#include STBIR__HEADER_FILENAME
6109+
6110+#define STBIR__vertical_channels 3
6111+#define STB_IMAGE_RESIZE_DO_VERTICALS
6112+#define STB_IMAGE_RESIZE_VERTICAL_CONTINUE
6113+#include STBIR__HEADER_FILENAME
6114+
6115+#define STBIR__vertical_channels 4
6116+#define STB_IMAGE_RESIZE_DO_VERTICALS
6117+#include STBIR__HEADER_FILENAME
6118+
6119+#define STBIR__vertical_channels 4
6120+#define STB_IMAGE_RESIZE_DO_VERTICALS
6121+#define STB_IMAGE_RESIZE_VERTICAL_CONTINUE
6122+#include STBIR__HEADER_FILENAME
6123+
6124+#define STBIR__vertical_channels 5
6125+#define STB_IMAGE_RESIZE_DO_VERTICALS
6126+#include STBIR__HEADER_FILENAME
6127+
6128+#define STBIR__vertical_channels 5
6129+#define STB_IMAGE_RESIZE_DO_VERTICALS
6130+#define STB_IMAGE_RESIZE_VERTICAL_CONTINUE
6131+#include STBIR__HEADER_FILENAME
6132+
6133+#define STBIR__vertical_channels 6
6134+#define STB_IMAGE_RESIZE_DO_VERTICALS
6135+#include STBIR__HEADER_FILENAME
6136+
6137+#define STBIR__vertical_channels 6
6138+#define STB_IMAGE_RESIZE_DO_VERTICALS
6139+#define STB_IMAGE_RESIZE_VERTICAL_CONTINUE
6140+#include STBIR__HEADER_FILENAME
6141+
6142+#define STBIR__vertical_channels 7
6143+#define STB_IMAGE_RESIZE_DO_VERTICALS
6144+#include STBIR__HEADER_FILENAME
6145+
6146+#define STBIR__vertical_channels 7
6147+#define STB_IMAGE_RESIZE_DO_VERTICALS
6148+#define STB_IMAGE_RESIZE_VERTICAL_CONTINUE
6149+#include STBIR__HEADER_FILENAME
6150+
6151+#define STBIR__vertical_channels 8
6152+#define STB_IMAGE_RESIZE_DO_VERTICALS
6153+#include STBIR__HEADER_FILENAME
6154+
6155+#define STBIR__vertical_channels 8
6156+#define STB_IMAGE_RESIZE_DO_VERTICALS
6157+#define STB_IMAGE_RESIZE_VERTICAL_CONTINUE
6158+#include STBIR__HEADER_FILENAME
6159+
6160+typedef void STBIR_VERTICAL_GATHERFUNC( float * output, float const * coeffs, float const ** inputs, float const * input0_end );
6161+
6162+static STBIR_VERTICAL_GATHERFUNC * stbir__vertical_gathers[ 8 ] =
6163+{
6164+ stbir__vertical_gather_with_1_coeffs,stbir__vertical_gather_with_2_coeffs,stbir__vertical_gather_with_3_coeffs,stbir__vertical_gather_with_4_coeffs,stbir__vertical_gather_with_5_coeffs,stbir__vertical_gather_with_6_coeffs,stbir__vertical_gather_with_7_coeffs,stbir__vertical_gather_with_8_coeffs
6165+};
6166+
6167+static STBIR_VERTICAL_GATHERFUNC * stbir__vertical_gathers_continues[ 8 ] =
6168+{
6169+ stbir__vertical_gather_with_1_coeffs_cont,stbir__vertical_gather_with_2_coeffs_cont,stbir__vertical_gather_with_3_coeffs_cont,stbir__vertical_gather_with_4_coeffs_cont,stbir__vertical_gather_with_5_coeffs_cont,stbir__vertical_gather_with_6_coeffs_cont,stbir__vertical_gather_with_7_coeffs_cont,stbir__vertical_gather_with_8_coeffs_cont
6170+};
6171+
6172+typedef void STBIR_VERTICAL_SCATTERFUNC( float ** outputs, float const * coeffs, float const * input, float const * input_end );
6173+
6174+static STBIR_VERTICAL_SCATTERFUNC * stbir__vertical_scatter_sets[ 8 ] =
6175+{
6176+ stbir__vertical_scatter_with_1_coeffs,stbir__vertical_scatter_with_2_coeffs,stbir__vertical_scatter_with_3_coeffs,stbir__vertical_scatter_with_4_coeffs,stbir__vertical_scatter_with_5_coeffs,stbir__vertical_scatter_with_6_coeffs,stbir__vertical_scatter_with_7_coeffs,stbir__vertical_scatter_with_8_coeffs
6177+};
6178+
6179+static STBIR_VERTICAL_SCATTERFUNC * stbir__vertical_scatter_blends[ 8 ] =
6180+{
6181+ stbir__vertical_scatter_with_1_coeffs_cont,stbir__vertical_scatter_with_2_coeffs_cont,stbir__vertical_scatter_with_3_coeffs_cont,stbir__vertical_scatter_with_4_coeffs_cont,stbir__vertical_scatter_with_5_coeffs_cont,stbir__vertical_scatter_with_6_coeffs_cont,stbir__vertical_scatter_with_7_coeffs_cont,stbir__vertical_scatter_with_8_coeffs_cont
6182+};
6183+
6184+
6185+static void stbir__encode_scanline( stbir__info const * stbir_info, void *output_buffer_data, float * encode_buffer, int row STBIR_ONLY_PROFILE_GET_SPLIT_INFO )
6186+{
6187+ int num_pixels = stbir_info->horizontal.scale_info.output_sub_size;
6188+ int channels = stbir_info->channels;
6189+ int width_times_channels = num_pixels * channels;
6190+ void * output_buffer;
6191+
6192+ // un-alpha weight if we need to
6193+ if ( stbir_info->alpha_unweight )
6194+ {
6195+ STBIR_PROFILE_START( unalpha );
6196+ stbir_info->alpha_unweight( encode_buffer, width_times_channels );
6197+ STBIR_PROFILE_END( unalpha );
6198+ }
6199+
6200+ // write directly into output by default
6201+ output_buffer = output_buffer_data;
6202+
6203+ // if we have an output callback, we first convert the decode buffer in place (and then hand that to the callback)
6204+ if ( stbir_info->out_pixels_cb )
6205+ output_buffer = encode_buffer;
6206+
6207+ STBIR_PROFILE_START( encode );
6208+ // convert into the output buffer
6209+ stbir_info->encode_pixels( output_buffer, width_times_channels, encode_buffer );
6210+ STBIR_PROFILE_END( encode );
6211+
6212+ // if we have an output callback, call it to send the data
6213+ if ( stbir_info->out_pixels_cb )
6214+ stbir_info->out_pixels_cb( output_buffer, num_pixels, row, stbir_info->user_data );
6215+}
6216+
6217+
6218+// Get the ring buffer pointer for an index
6219+static float* stbir__get_ring_buffer_entry(stbir__info const * stbir_info, stbir__per_split_info const * split_info, int index )
6220+{
6221+ STBIR_ASSERT( index < stbir_info->ring_buffer_num_entries );
6222+
6223+ #ifdef STBIR__SEPARATE_ALLOCATIONS
6224+ return split_info->ring_buffers[ index ];
6225+ #else
6226+ return (float*) ( ( (char*) split_info->ring_buffer ) + ( index * stbir_info->ring_buffer_length_bytes ) );
6227+ #endif
6228+}
6229+
6230+// Get the specified scan line from the ring buffer
6231+static float* stbir__get_ring_buffer_scanline(stbir__info const * stbir_info, stbir__per_split_info const * split_info, int get_scanline)
6232+{
6233+ int ring_buffer_index = (split_info->ring_buffer_begin_index + (get_scanline - split_info->ring_buffer_first_scanline)) % stbir_info->ring_buffer_num_entries;
6234+ return stbir__get_ring_buffer_entry( stbir_info, split_info, ring_buffer_index );
6235+}
6236+
6237+static void stbir__resample_horizontal_gather(stbir__info const * stbir_info, float* output_buffer, float const * input_buffer STBIR_ONLY_PROFILE_GET_SPLIT_INFO )
6238+{
6239+ float const * decode_buffer = input_buffer - ( stbir_info->scanline_extents.conservative.n0 * stbir_info->effective_channels );
6240+
6241+ STBIR_PROFILE_START( horizontal );
6242+ if ( ( stbir_info->horizontal.filter_enum == STBIR_FILTER_POINT_SAMPLE ) && ( stbir_info->horizontal.scale_info.scale == 1.0f ) )
6243+ STBIR_MEMCPY( output_buffer, input_buffer, stbir_info->horizontal.scale_info.output_sub_size * sizeof( float ) * stbir_info->effective_channels );
6244+ else
6245+ stbir_info->horizontal_gather_channels( output_buffer, stbir_info->horizontal.scale_info.output_sub_size, decode_buffer, stbir_info->horizontal.contributors, stbir_info->horizontal.coefficients, stbir_info->horizontal.coefficient_width );
6246+ STBIR_PROFILE_END( horizontal );
6247+}
6248+
6249+static void stbir__resample_vertical_gather(stbir__info const * stbir_info, stbir__per_split_info* split_info, int n, int contrib_n0, int contrib_n1, float const * vertical_coefficients )
6250+{
6251+ float* encode_buffer = split_info->vertical_buffer;
6252+ float* decode_buffer = split_info->decode_buffer;
6253+ int vertical_first = stbir_info->vertical_first;
6254+ int width = (vertical_first) ? ( stbir_info->scanline_extents.conservative.n1-stbir_info->scanline_extents.conservative.n0+1 ) : stbir_info->horizontal.scale_info.output_sub_size;
6255+ int width_times_channels = stbir_info->effective_channels * width;
6256+
6257+ STBIR_ASSERT( stbir_info->vertical.is_gather );
6258+
6259+ // loop over the contributing scanlines and scale into the buffer
6260+ STBIR_PROFILE_START( vertical );
6261+ {
6262+ int k = 0, total = contrib_n1 - contrib_n0 + 1;
6263+ STBIR_ASSERT( total > 0 );
6264+ do {
6265+ float const * inputs[8];
6266+ int i, cnt = total; if ( cnt > 8 ) cnt = 8;
6267+ for( i = 0 ; i < cnt ; i++ )
6268+ inputs[ i ] = stbir__get_ring_buffer_scanline(stbir_info, split_info, k+i+contrib_n0 );
6269+
6270+ // call the N scanlines at a time function (up to 8 scanlines of blending at once)
6271+ ((k==0)?stbir__vertical_gathers:stbir__vertical_gathers_continues)[cnt-1]( (vertical_first) ? decode_buffer : encode_buffer, vertical_coefficients + k, inputs, inputs[0] + width_times_channels );
6272+ k += cnt;
6273+ total -= cnt;
6274+ } while ( total );
6275+ }
6276+ STBIR_PROFILE_END( vertical );
6277+
6278+ if ( vertical_first )
6279+ {
6280+ // Now resample the gathered vertical data in the horizontal axis into the encode buffer
6281+ decode_buffer[ width_times_channels ] = 0.0f; // clear two over for horizontals with a remnant of 3
6282+ decode_buffer[ width_times_channels+1 ] = 0.0f;
6283+ stbir__resample_horizontal_gather(stbir_info, encode_buffer, decode_buffer STBIR_ONLY_PROFILE_SET_SPLIT_INFO );
6284+ }
6285+
6286+ stbir__encode_scanline( stbir_info, ( (char *) stbir_info->output_data ) + ((size_t)n * (size_t)stbir_info->output_stride_bytes),
6287+ encode_buffer, n STBIR_ONLY_PROFILE_SET_SPLIT_INFO );
6288+}
6289+
6290+static void stbir__decode_and_resample_for_vertical_gather_loop(stbir__info const * stbir_info, stbir__per_split_info* split_info, int n)
6291+{
6292+ int ring_buffer_index;
6293+ float* ring_buffer;
6294+
6295+ // Decode the nth scanline from the source image into the decode buffer.
6296+ stbir__decode_scanline( stbir_info, n, split_info->decode_buffer STBIR_ONLY_PROFILE_SET_SPLIT_INFO );
6297+
6298+ // update new end scanline
6299+ split_info->ring_buffer_last_scanline = n;
6300+
6301+ // get ring buffer
6302+ ring_buffer_index = (split_info->ring_buffer_begin_index + (split_info->ring_buffer_last_scanline - split_info->ring_buffer_first_scanline)) % stbir_info->ring_buffer_num_entries;
6303+ ring_buffer = stbir__get_ring_buffer_entry(stbir_info, split_info, ring_buffer_index);
6304+
6305+ // Now resample it into the ring buffer.
6306+ stbir__resample_horizontal_gather( stbir_info, ring_buffer, split_info->decode_buffer STBIR_ONLY_PROFILE_SET_SPLIT_INFO );
6307+
6308+ // Now it's sitting in the ring buffer ready to be used as source for the vertical sampling.
6309+}
6310+
6311+static void stbir__vertical_gather_loop( stbir__info const * stbir_info, stbir__per_split_info* split_info, int split_count )
6312+{
6313+ int y, start_output_y, end_output_y;
6314+ stbir__contributors* vertical_contributors = stbir_info->vertical.contributors;
6315+ float const * vertical_coefficients = stbir_info->vertical.coefficients;
6316+
6317+ STBIR_ASSERT( stbir_info->vertical.is_gather );
6318+
6319+ start_output_y = split_info->start_output_y;
6320+ end_output_y = split_info[split_count-1].end_output_y;
6321+
6322+ vertical_contributors += start_output_y;
6323+ vertical_coefficients += start_output_y * stbir_info->vertical.coefficient_width;
6324+
6325+ // initialize the ring buffer for gathering
6326+ split_info->ring_buffer_begin_index = 0;
6327+ split_info->ring_buffer_first_scanline = vertical_contributors->n0;
6328+ split_info->ring_buffer_last_scanline = split_info->ring_buffer_first_scanline - 1; // means "empty"
6329+
6330+ for (y = start_output_y; y < end_output_y; y++)
6331+ {
6332+ int in_first_scanline, in_last_scanline;
6333+
6334+ in_first_scanline = vertical_contributors->n0;
6335+ in_last_scanline = vertical_contributors->n1;
6336+
6337+ // make sure the indexing hasn't broken
6338+ STBIR_ASSERT( in_first_scanline >= split_info->ring_buffer_first_scanline );
6339+
6340+ // Load in new scanlines
6341+ while (in_last_scanline > split_info->ring_buffer_last_scanline)
6342+ {
6343+ STBIR_ASSERT( ( split_info->ring_buffer_last_scanline - split_info->ring_buffer_first_scanline + 1 ) <= stbir_info->ring_buffer_num_entries );
6344+
6345+ // make sure there was room in the ring buffer when we add new scanlines
6346+ if ( ( split_info->ring_buffer_last_scanline - split_info->ring_buffer_first_scanline + 1 ) == stbir_info->ring_buffer_num_entries )
6347+ {
6348+ split_info->ring_buffer_first_scanline++;
6349+ split_info->ring_buffer_begin_index++;
6350+ }
6351+
6352+ if ( stbir_info->vertical_first )
6353+ {
6354+ float * ring_buffer = stbir__get_ring_buffer_scanline( stbir_info, split_info, ++split_info->ring_buffer_last_scanline );
6355+ // Decode the nth scanline from the source image into the decode buffer.
6356+ stbir__decode_scanline( stbir_info, split_info->ring_buffer_last_scanline, ring_buffer STBIR_ONLY_PROFILE_SET_SPLIT_INFO );
6357+ }
6358+ else
6359+ {
6360+ stbir__decode_and_resample_for_vertical_gather_loop(stbir_info, split_info, split_info->ring_buffer_last_scanline + 1);
6361+ }
6362+ }
6363+
6364+ // Now all buffers should be ready to write a row of vertical sampling, so do it.
6365+ stbir__resample_vertical_gather(stbir_info, split_info, y, in_first_scanline, in_last_scanline, vertical_coefficients );
6366+
6367+ ++vertical_contributors;
6368+ vertical_coefficients += stbir_info->vertical.coefficient_width;
6369+ }
6370+}
6371+
6372+#define STBIR__FLOAT_EMPTY_MARKER 3.0e+38F
6373+#define STBIR__FLOAT_BUFFER_IS_EMPTY(ptr) ((ptr)[0]==STBIR__FLOAT_EMPTY_MARKER)
6374+
6375+static void stbir__encode_first_scanline_from_scatter(stbir__info const * stbir_info, stbir__per_split_info* split_info)
6376+{
6377+ // evict a scanline out into the output buffer
6378+ float* ring_buffer_entry = stbir__get_ring_buffer_entry(stbir_info, split_info, split_info->ring_buffer_begin_index );
6379+
6380+ // dump the scanline out
6381+ stbir__encode_scanline( stbir_info, ( (char *)stbir_info->output_data ) + ( (size_t)split_info->ring_buffer_first_scanline * (size_t)stbir_info->output_stride_bytes ), ring_buffer_entry, split_info->ring_buffer_first_scanline STBIR_ONLY_PROFILE_SET_SPLIT_INFO );
6382+
6383+ // mark it as empty
6384+ ring_buffer_entry[ 0 ] = STBIR__FLOAT_EMPTY_MARKER;
6385+
6386+ // advance the first scanline
6387+ split_info->ring_buffer_first_scanline++;
6388+ if ( ++split_info->ring_buffer_begin_index == stbir_info->ring_buffer_num_entries )
6389+ split_info->ring_buffer_begin_index = 0;
6390+}
6391+
6392+static void stbir__horizontal_resample_and_encode_first_scanline_from_scatter(stbir__info const * stbir_info, stbir__per_split_info* split_info)
6393+{
6394+ // evict a scanline out into the output buffer
6395+
6396+ float* ring_buffer_entry = stbir__get_ring_buffer_entry(stbir_info, split_info, split_info->ring_buffer_begin_index );
6397+
6398+ // Now resample it into the buffer.
6399+ stbir__resample_horizontal_gather( stbir_info, split_info->vertical_buffer, ring_buffer_entry STBIR_ONLY_PROFILE_SET_SPLIT_INFO );
6400+
6401+ // dump the scanline out
6402+ stbir__encode_scanline( stbir_info, ( (char *)stbir_info->output_data ) + ( (size_t)split_info->ring_buffer_first_scanline * (size_t)stbir_info->output_stride_bytes ), split_info->vertical_buffer, split_info->ring_buffer_first_scanline STBIR_ONLY_PROFILE_SET_SPLIT_INFO );
6403+
6404+ // mark it as empty
6405+ ring_buffer_entry[ 0 ] = STBIR__FLOAT_EMPTY_MARKER;
6406+
6407+ // advance the first scanline
6408+ split_info->ring_buffer_first_scanline++;
6409+ if ( ++split_info->ring_buffer_begin_index == stbir_info->ring_buffer_num_entries )
6410+ split_info->ring_buffer_begin_index = 0;
6411+}
6412+
6413+static void stbir__resample_vertical_scatter(stbir__info const * stbir_info, stbir__per_split_info* split_info, int n0, int n1, float const * vertical_coefficients, float const * vertical_buffer, float const * vertical_buffer_end )
6414+{
6415+ STBIR_ASSERT( !stbir_info->vertical.is_gather );
6416+
6417+ STBIR_PROFILE_START( vertical );
6418+ {
6419+ int k = 0, total = n1 - n0 + 1;
6420+ STBIR_ASSERT( total > 0 );
6421+ do {
6422+ float * outputs[8];
6423+ int i, n = total; if ( n > 8 ) n = 8;
6424+ for( i = 0 ; i < n ; i++ )
6425+ {
6426+ outputs[ i ] = stbir__get_ring_buffer_scanline(stbir_info, split_info, k+i+n0 );
6427+ if ( ( i ) && ( STBIR__FLOAT_BUFFER_IS_EMPTY( outputs[i] ) != STBIR__FLOAT_BUFFER_IS_EMPTY( outputs[0] ) ) ) // make sure runs are of the same type
6428+ {
6429+ n = i;
6430+ break;
6431+ }
6432+ }
6433+ // call the scatter to N scanlines at a time function (up to 8 scanlines of scattering at once)
6434+ ((STBIR__FLOAT_BUFFER_IS_EMPTY( outputs[0] ))?stbir__vertical_scatter_sets:stbir__vertical_scatter_blends)[n-1]( outputs, vertical_coefficients + k, vertical_buffer, vertical_buffer_end );
6435+ k += n;
6436+ total -= n;
6437+ } while ( total );
6438+ }
6439+
6440+ STBIR_PROFILE_END( vertical );
6441+}
6442+
6443+typedef void stbir__handle_scanline_for_scatter_func(stbir__info const * stbir_info, stbir__per_split_info* split_info);
6444+
6445+static void stbir__vertical_scatter_loop( stbir__info const * stbir_info, stbir__per_split_info* split_info, int split_count )
6446+{
6447+ int y, start_output_y, end_output_y, start_input_y, end_input_y;
6448+ stbir__contributors* vertical_contributors = stbir_info->vertical.contributors;
6449+ float const * vertical_coefficients = stbir_info->vertical.coefficients;
6450+ stbir__handle_scanline_for_scatter_func * handle_scanline_for_scatter;
6451+ void * scanline_scatter_buffer;
6452+ void * scanline_scatter_buffer_end;
6453+ int on_first_input_y, last_input_y;
6454+ int width = (stbir_info->vertical_first) ? ( stbir_info->scanline_extents.conservative.n1-stbir_info->scanline_extents.conservative.n0+1 ) : stbir_info->horizontal.scale_info.output_sub_size;
6455+ int width_times_channels = stbir_info->effective_channels * width;
6456+
6457+ STBIR_ASSERT( !stbir_info->vertical.is_gather );
6458+
6459+ start_output_y = split_info->start_output_y;
6460+ end_output_y = split_info[split_count-1].end_output_y; // may do multiple split counts
6461+
6462+ start_input_y = split_info->start_input_y;
6463+ end_input_y = split_info[split_count-1].end_input_y;
6464+
6465+ // adjust for starting offset start_input_y
6466+ y = start_input_y + stbir_info->vertical.filter_pixel_margin;
6467+ vertical_contributors += y ;
6468+ vertical_coefficients += stbir_info->vertical.coefficient_width * y;
6469+
6470+ if ( stbir_info->vertical_first )
6471+ {
6472+ handle_scanline_for_scatter = stbir__horizontal_resample_and_encode_first_scanline_from_scatter;
6473+ scanline_scatter_buffer = split_info->decode_buffer;
6474+ scanline_scatter_buffer_end = ( (char*) scanline_scatter_buffer ) + sizeof( float ) * stbir_info->effective_channels * (stbir_info->scanline_extents.conservative.n1-stbir_info->scanline_extents.conservative.n0+1);
6475+ }
6476+ else
6477+ {
6478+ handle_scanline_for_scatter = stbir__encode_first_scanline_from_scatter;
6479+ scanline_scatter_buffer = split_info->vertical_buffer;
6480+ scanline_scatter_buffer_end = ( (char*) scanline_scatter_buffer ) + sizeof( float ) * stbir_info->effective_channels * stbir_info->horizontal.scale_info.output_sub_size;
6481+ }
6482+
6483+ // initialize the ring buffer for scattering
6484+ split_info->ring_buffer_first_scanline = start_output_y;
6485+ split_info->ring_buffer_last_scanline = -1;
6486+ split_info->ring_buffer_begin_index = -1;
6487+
6488+ // mark all the buffers as empty to start
6489+ for( y = 0 ; y < stbir_info->ring_buffer_num_entries ; y++ )
6490+ {
6491+ float * decode_buffer = stbir__get_ring_buffer_entry( stbir_info, split_info, y );
6492+ decode_buffer[ width_times_channels ] = 0.0f; // clear two over for horizontals with a remnant of 3
6493+ decode_buffer[ width_times_channels+1 ] = 0.0f;
6494+ decode_buffer[0] = STBIR__FLOAT_EMPTY_MARKER; // only used on scatter
6495+ }
6496+
6497+ // do the loop in input space
6498+ on_first_input_y = 1; last_input_y = start_input_y;
6499+ for (y = start_input_y ; y < end_input_y; y++)
6500+ {
6501+ int out_first_scanline, out_last_scanline;
6502+
6503+ out_first_scanline = vertical_contributors->n0;
6504+ out_last_scanline = vertical_contributors->n1;
6505+
6506+ STBIR_ASSERT(out_last_scanline - out_first_scanline + 1 <= stbir_info->ring_buffer_num_entries);
6507+
6508+ if ( ( out_last_scanline >= out_first_scanline ) && ( ( ( out_first_scanline >= start_output_y ) && ( out_first_scanline < end_output_y ) ) || ( ( out_last_scanline >= start_output_y ) && ( out_last_scanline < end_output_y ) ) ) )
6509+ {
6510+ float const * vc = vertical_coefficients;
6511+
6512+ // keep track of the range actually seen for the next resize
6513+ last_input_y = y;
6514+ if ( ( on_first_input_y ) && ( y > start_input_y ) )
6515+ split_info->start_input_y = y;
6516+ on_first_input_y = 0;
6517+
6518+ // clip the region
6519+ if ( out_first_scanline < start_output_y )
6520+ {
6521+ vc += start_output_y - out_first_scanline;
6522+ out_first_scanline = start_output_y;
6523+ }
6524+
6525+ if ( out_last_scanline >= end_output_y )
6526+ out_last_scanline = end_output_y - 1;
6527+
6528+ // if very first scanline, init the index
6529+ if (split_info->ring_buffer_begin_index < 0)
6530+ split_info->ring_buffer_begin_index = out_first_scanline - start_output_y;
6531+
6532+ STBIR_ASSERT( split_info->ring_buffer_begin_index <= out_first_scanline );
6533+
6534+ // Decode the nth scanline from the source image into the decode buffer.
6535+ stbir__decode_scanline( stbir_info, y, split_info->decode_buffer STBIR_ONLY_PROFILE_SET_SPLIT_INFO );
6536+
6537+ // When horizontal first, we resample horizontally into the vertical buffer before we scatter it out
6538+ if ( !stbir_info->vertical_first )
6539+ stbir__resample_horizontal_gather( stbir_info, split_info->vertical_buffer, split_info->decode_buffer STBIR_ONLY_PROFILE_SET_SPLIT_INFO );
6540+
6541+ // Now it's sitting in the buffer ready to be distributed into the ring buffers.
6542+
6543+ // evict from the ringbuffer, if we need are full
6544+ if ( ( ( split_info->ring_buffer_last_scanline - split_info->ring_buffer_first_scanline + 1 ) == stbir_info->ring_buffer_num_entries ) &&
6545+ ( out_last_scanline > split_info->ring_buffer_last_scanline ) )
6546+ handle_scanline_for_scatter( stbir_info, split_info );
6547+
6548+ // Now the horizontal buffer is ready to write to all ring buffer rows, so do it.
6549+ stbir__resample_vertical_scatter(stbir_info, split_info, out_first_scanline, out_last_scanline, vc, (float*)scanline_scatter_buffer, (float*)scanline_scatter_buffer_end );
6550+
6551+ // update the end of the buffer
6552+ if ( out_last_scanline > split_info->ring_buffer_last_scanline )
6553+ split_info->ring_buffer_last_scanline = out_last_scanline;
6554+ }
6555+ ++vertical_contributors;
6556+ vertical_coefficients += stbir_info->vertical.coefficient_width;
6557+ }
6558+
6559+ // now evict the scanlines that are left over in the ring buffer
6560+ while ( split_info->ring_buffer_first_scanline < end_output_y )
6561+ handle_scanline_for_scatter(stbir_info, split_info);
6562+
6563+ // update the end_input_y if we do multiple resizes with the same data
6564+ ++last_input_y;
6565+ for( y = 0 ; y < split_count; y++ )
6566+ if ( split_info[y].end_input_y > last_input_y )
6567+ split_info[y].end_input_y = last_input_y;
6568+}
6569+
6570+
6571+static stbir__kernel_callback * stbir__builtin_kernels[] = { 0, stbir__filter_trapezoid, stbir__filter_triangle, stbir__filter_cubic, stbir__filter_catmullrom, stbir__filter_mitchell, stbir__filter_point };
6572+static stbir__support_callback * stbir__builtin_supports[] = { 0, stbir__support_trapezoid, stbir__support_one, stbir__support_two, stbir__support_two, stbir__support_two, stbir__support_zeropoint5 };
6573+
6574+static void stbir__set_sampler(stbir__sampler * samp, stbir_filter filter, stbir__kernel_callback * kernel, stbir__support_callback * support, stbir_edge edge, stbir__scale_info * scale_info, int always_gather, void * user_data )
6575+{
6576+ // set filter
6577+ if (filter == 0)
6578+ {
6579+ filter = STBIR_DEFAULT_FILTER_DOWNSAMPLE; // default to downsample
6580+ if (scale_info->scale >= ( 1.0f - stbir__small_float ) )
6581+ {
6582+ if ( (scale_info->scale <= ( 1.0f + stbir__small_float ) ) && ( STBIR_CEILF(scale_info->pixel_shift) == scale_info->pixel_shift ) )
6583+ filter = STBIR_FILTER_POINT_SAMPLE;
6584+ else
6585+ filter = STBIR_DEFAULT_FILTER_UPSAMPLE;
6586+ }
6587+ }
6588+ samp->filter_enum = filter;
6589+
6590+ STBIR_ASSERT(samp->filter_enum != 0);
6591+ STBIR_ASSERT((unsigned)samp->filter_enum < STBIR_FILTER_OTHER);
6592+ samp->filter_kernel = stbir__builtin_kernels[ filter ];
6593+ samp->filter_support = stbir__builtin_supports[ filter ];
6594+
6595+ if ( kernel && support )
6596+ {
6597+ samp->filter_kernel = kernel;
6598+ samp->filter_support = support;
6599+ samp->filter_enum = STBIR_FILTER_OTHER;
6600+ }
6601+
6602+ samp->edge = edge;
6603+ samp->filter_pixel_width = stbir__get_filter_pixel_width (samp->filter_support, scale_info->scale, user_data );
6604+ // Gather is always better, but in extreme downsamples, you have to most or all of the data in memory
6605+ // For horizontal, we always have all the pixels, so we always use gather here (always_gather==1).
6606+ // For vertical, we use gather if scaling up (which means we will have samp->filter_pixel_width
6607+ // scanlines in memory at once).
6608+ samp->is_gather = 0;
6609+ if ( scale_info->scale >= ( 1.0f - stbir__small_float ) )
6610+ samp->is_gather = 1;
6611+ else if ( ( always_gather ) || ( samp->filter_pixel_width <= STBIR_FORCE_GATHER_FILTER_SCANLINES_AMOUNT ) )
6612+ samp->is_gather = 2;
6613+
6614+ // pre calculate stuff based on the above
6615+ samp->coefficient_width = stbir__get_coefficient_width(samp, samp->is_gather, user_data);
6616+
6617+ // filter_pixel_width is the conservative size in pixels of input that affect an output pixel.
6618+ // In rare cases (only with 2 pix to 1 pix with the default filters), it's possible that the
6619+ // filter will extend before or after the scanline beyond just one extra entire copy of the
6620+ // scanline (we would hit the edge twice). We don't let you do that, so we clamp the total
6621+ // width to 3x the total of input pixel (once for the scanline, once for the left side
6622+ // overhang, and once for the right side). We only do this for edge mode, since the other
6623+ // modes can just re-edge clamp back in again.
6624+ if ( edge == STBIR_EDGE_WRAP )
6625+ if ( samp->filter_pixel_width > ( scale_info->input_full_size * 3 ) )
6626+ samp->filter_pixel_width = scale_info->input_full_size * 3;
6627+
6628+ // This is how much to expand buffers to account for filters seeking outside
6629+ // the image boundaries.
6630+ samp->filter_pixel_margin = samp->filter_pixel_width / 2;
6631+
6632+ // filter_pixel_margin is the amount that this filter can overhang on just one side of either
6633+ // end of the scanline (left or the right). Since we only allow you to overhang 1 scanline's
6634+ // worth of pixels, we clamp this one side of overhang to the input scanline size. Again,
6635+ // this clamping only happens in rare cases with the default filters (2 pix to 1 pix).
6636+ if ( edge == STBIR_EDGE_WRAP )
6637+ if ( samp->filter_pixel_margin > scale_info->input_full_size )
6638+ samp->filter_pixel_margin = scale_info->input_full_size;
6639+
6640+ samp->num_contributors = stbir__get_contributors(samp, samp->is_gather);
6641+
6642+ samp->contributors_size = samp->num_contributors * sizeof(stbir__contributors);
6643+ samp->coefficients_size = samp->num_contributors * samp->coefficient_width * sizeof(float) + sizeof(float)*STBIR_INPUT_CALLBACK_PADDING; // extra sizeof(float) is padding
6644+
6645+ samp->gather_prescatter_contributors = 0;
6646+ samp->gather_prescatter_coefficients = 0;
6647+ if ( samp->is_gather == 0 )
6648+ {
6649+ samp->gather_prescatter_coefficient_width = samp->filter_pixel_width;
6650+ samp->gather_prescatter_num_contributors = stbir__get_contributors(samp, 2);
6651+ samp->gather_prescatter_contributors_size = samp->gather_prescatter_num_contributors * sizeof(stbir__contributors);
6652+ samp->gather_prescatter_coefficients_size = samp->gather_prescatter_num_contributors * samp->gather_prescatter_coefficient_width * sizeof(float);
6653+ }
6654+}
6655+
6656+static void stbir__get_conservative_extents( stbir__sampler * samp, stbir__contributors * range, void * user_data )
6657+{
6658+ float scale = samp->scale_info.scale;
6659+ float out_shift = samp->scale_info.pixel_shift;
6660+ stbir__support_callback * support = samp->filter_support;
6661+ int input_full_size = samp->scale_info.input_full_size;
6662+ stbir_edge edge = samp->edge;
6663+ float inv_scale = samp->scale_info.inv_scale;
6664+
6665+ STBIR_ASSERT( samp->is_gather != 0 );
6666+
6667+ if ( samp->is_gather == 1 )
6668+ {
6669+ int in_first_pixel, in_last_pixel;
6670+ float out_filter_radius = support(inv_scale, user_data) * scale;
6671+
6672+ stbir__calculate_in_pixel_range( &in_first_pixel, &in_last_pixel, 0.5, out_filter_radius, inv_scale, out_shift, input_full_size, edge );
6673+ range->n0 = in_first_pixel;
6674+ stbir__calculate_in_pixel_range( &in_first_pixel, &in_last_pixel, ( (float)(samp->scale_info.output_sub_size-1) ) + 0.5f, out_filter_radius, inv_scale, out_shift, input_full_size, edge );
6675+ range->n1 = in_last_pixel;
6676+ }
6677+ else if ( samp->is_gather == 2 ) // downsample gather, refine
6678+ {
6679+ float in_pixels_radius = support(scale, user_data) * inv_scale;
6680+ int filter_pixel_margin = samp->filter_pixel_margin;
6681+ int output_sub_size = samp->scale_info.output_sub_size;
6682+ int input_end;
6683+ int n;
6684+ int in_first_pixel, in_last_pixel;
6685+
6686+ // get a conservative area of the input range
6687+ stbir__calculate_in_pixel_range( &in_first_pixel, &in_last_pixel, 0, 0, inv_scale, out_shift, input_full_size, edge );
6688+ range->n0 = in_first_pixel;
6689+ stbir__calculate_in_pixel_range( &in_first_pixel, &in_last_pixel, (float)output_sub_size, 0, inv_scale, out_shift, input_full_size, edge );
6690+ range->n1 = in_last_pixel;
6691+
6692+ // now go through the margin to the start of area to find bottom
6693+ n = range->n0 + 1;
6694+ input_end = -filter_pixel_margin;
6695+ while( n >= input_end )
6696+ {
6697+ int out_first_pixel, out_last_pixel;
6698+ stbir__calculate_out_pixel_range( &out_first_pixel, &out_last_pixel, ((float)n)+0.5f, in_pixels_radius, scale, out_shift, output_sub_size );
6699+ if ( out_first_pixel > out_last_pixel )
6700+ break;
6701+
6702+ if ( ( out_first_pixel < output_sub_size ) || ( out_last_pixel >= 0 ) )
6703+ range->n0 = n;
6704+ --n;
6705+ }
6706+
6707+ // now go through the end of the area through the margin to find top
6708+ n = range->n1 - 1;
6709+ input_end = n + 1 + filter_pixel_margin;
6710+ while( n <= input_end )
6711+ {
6712+ int out_first_pixel, out_last_pixel;
6713+ stbir__calculate_out_pixel_range( &out_first_pixel, &out_last_pixel, ((float)n)+0.5f, in_pixels_radius, scale, out_shift, output_sub_size );
6714+ if ( out_first_pixel > out_last_pixel )
6715+ break;
6716+ if ( ( out_first_pixel < output_sub_size ) || ( out_last_pixel >= 0 ) )
6717+ range->n1 = n;
6718+ ++n;
6719+ }
6720+ }
6721+
6722+ if ( samp->edge == STBIR_EDGE_WRAP )
6723+ {
6724+ // if we are wrapping, and we are very close to the image size (so the edges might merge), just use the scanline up to the edge
6725+ if ( ( range->n0 > 0 ) && ( range->n1 >= input_full_size ) )
6726+ {
6727+ int marg = range->n1 - input_full_size + 1;
6728+ if ( ( marg + STBIR__MERGE_RUNS_PIXEL_THRESHOLD ) >= range->n0 )
6729+ range->n0 = 0;
6730+ }
6731+ if ( ( range->n0 < 0 ) && ( range->n1 < (input_full_size-1) ) )
6732+ {
6733+ int marg = -range->n0;
6734+ if ( ( input_full_size - marg - STBIR__MERGE_RUNS_PIXEL_THRESHOLD - 1 ) <= range->n1 )
6735+ range->n1 = input_full_size - 1;
6736+ }
6737+ }
6738+ else
6739+ {
6740+ // for non-edge-wrap modes, we never read over the edge, so clamp
6741+ if ( range->n0 < 0 )
6742+ range->n0 = 0;
6743+ if ( range->n1 >= input_full_size )
6744+ range->n1 = input_full_size - 1;
6745+ }
6746+}
6747+
6748+static void stbir__get_split_info( stbir__per_split_info* split_info, int splits, int output_height, int vertical_pixel_margin, int input_full_height, int is_gather, stbir__contributors * contribs )
6749+{
6750+ int i, cur;
6751+ int left = output_height;
6752+
6753+ cur = 0;
6754+ for( i = 0 ; i < splits ; i++ )
6755+ {
6756+ int each;
6757+
6758+ split_info[i].start_output_y = cur;
6759+ each = left / ( splits - i );
6760+ split_info[i].end_output_y = cur + each;
6761+
6762+ // ok, when we are gathering, we need to make sure we are starting on a y offset that doesn't have
6763+ // a "special" set of coefficients. Basically, with exactly the right filter at exactly the right
6764+ // resize at exactly the right phase, some of the coefficents can be zero. When they are zero, we
6765+ // don't process them at all. But this leads to a tricky thing with the thread splits, where we
6766+ // might have a set of two coeffs like this for example: (4,4) and (3,6). The 4,4 means there was
6767+ // just one single coeff because things worked out perfectly (normally, they all have 4 coeffs
6768+ // like the range 3,6. The problem is that if we start right on the (4,4) on a brand new thread,
6769+ // then when we get to (3,6), we don't have the "3" sample in memory (because we didn't load
6770+ // it on the initial (4,4) range because it didn't have a 3 (we only add new samples that are
6771+ // larger than our existing samples - it's just how the eviction works). So, our solution here
6772+ // is pretty simple, if we start right on a range that has samples that start earlier, then we
6773+ // simply bump up our previous thread split range to include it, and then start this threads
6774+ // range with the smaller sample. It just moves one scanline from one thread split to another,
6775+ // so that we end with the unusual one, instead of start with it. To do this, we check 2-4
6776+ // sample at each thread split start and then occassionally move them.
6777+
6778+ if ( ( is_gather ) && ( i ) )
6779+ {
6780+ stbir__contributors * small_contribs;
6781+ int j, smallest, stop, start_n0;
6782+ stbir__contributors * split_contribs = contribs + cur;
6783+
6784+ // scan for a max of 3x the filter width or until the next thread split
6785+ stop = vertical_pixel_margin * 3;
6786+ if ( each < stop )
6787+ stop = each;
6788+
6789+ // loops a few times before early out
6790+ smallest = 0;
6791+ small_contribs = split_contribs;
6792+ start_n0 = small_contribs->n0;
6793+ for( j = 1 ; j <= stop ; j++ )
6794+ {
6795+ ++split_contribs;
6796+ if ( split_contribs->n0 > start_n0 )
6797+ break;
6798+ if ( split_contribs->n0 < small_contribs->n0 )
6799+ {
6800+ small_contribs = split_contribs;
6801+ smallest = j;
6802+ }
6803+ }
6804+
6805+ split_info[i-1].end_output_y += smallest;
6806+ split_info[i].start_output_y += smallest;
6807+ }
6808+
6809+ cur += each;
6810+ left -= each;
6811+
6812+ // scatter range (updated to minimum as you run it)
6813+ split_info[i].start_input_y = -vertical_pixel_margin;
6814+ split_info[i].end_input_y = input_full_height + vertical_pixel_margin;
6815+ }
6816+}
6817+
6818+static void stbir__free_internal_mem( stbir__info *info )
6819+{
6820+ #define STBIR__FREE_AND_CLEAR( ptr ) { if ( ptr ) { void * p = (ptr); (ptr) = 0; STBIR_FREE( p, info->user_data); } }
6821+
6822+ if ( info )
6823+ {
6824+ #ifndef STBIR__SEPARATE_ALLOCATIONS
6825+ STBIR__FREE_AND_CLEAR( info->alloced_mem );
6826+ #else
6827+ int i,j;
6828+
6829+ if ( ( info->vertical.gather_prescatter_contributors ) && ( (void*)info->vertical.gather_prescatter_contributors != (void*)info->split_info[0].decode_buffer ) )
6830+ {
6831+ STBIR__FREE_AND_CLEAR( info->vertical.gather_prescatter_coefficients );
6832+ STBIR__FREE_AND_CLEAR( info->vertical.gather_prescatter_contributors );
6833+ }
6834+ for( i = 0 ; i < info->splits ; i++ )
6835+ {
6836+ for( j = 0 ; j < info->alloc_ring_buffer_num_entries ; j++ )
6837+ {
6838+ #ifdef STBIR_SIMD8
6839+ if ( info->effective_channels == 3 )
6840+ --info->split_info[i].ring_buffers[j]; // avx in 3 channel mode needs one float at the start of the buffer
6841+ #endif
6842+ STBIR__FREE_AND_CLEAR( info->split_info[i].ring_buffers[j] );
6843+ }
6844+
6845+ #ifdef STBIR_SIMD8
6846+ if ( info->effective_channels == 3 )
6847+ --info->split_info[i].decode_buffer; // avx in 3 channel mode needs one float at the start of the buffer
6848+ #endif
6849+ STBIR__FREE_AND_CLEAR( info->split_info[i].decode_buffer );
6850+ STBIR__FREE_AND_CLEAR( info->split_info[i].ring_buffers );
6851+ STBIR__FREE_AND_CLEAR( info->split_info[i].vertical_buffer );
6852+ }
6853+ STBIR__FREE_AND_CLEAR( info->split_info );
6854+ if ( info->vertical.coefficients != info->horizontal.coefficients )
6855+ {
6856+ STBIR__FREE_AND_CLEAR( info->vertical.coefficients );
6857+ STBIR__FREE_AND_CLEAR( info->vertical.contributors );
6858+ }
6859+ STBIR__FREE_AND_CLEAR( info->horizontal.coefficients );
6860+ STBIR__FREE_AND_CLEAR( info->horizontal.contributors );
6861+ STBIR__FREE_AND_CLEAR( info->alloced_mem );
6862+ STBIR_FREE( info, info->user_data );
6863+ #endif
6864+ }
6865+
6866+ #undef STBIR__FREE_AND_CLEAR
6867+}
6868+
6869+static int stbir__get_max_split( int splits, int height )
6870+{
6871+ int i;
6872+ int max = 0;
6873+
6874+ for( i = 0 ; i < splits ; i++ )
6875+ {
6876+ int each = height / ( splits - i );
6877+ if ( each > max )
6878+ max = each;
6879+ height -= each;
6880+ }
6881+ return max;
6882+}
6883+
6884+static stbir__horizontal_gather_channels_func ** stbir__horizontal_gather_n_coeffs_funcs[8] =
6885+{
6886+ 0, stbir__horizontal_gather_1_channels_with_n_coeffs_funcs, stbir__horizontal_gather_2_channels_with_n_coeffs_funcs, stbir__horizontal_gather_3_channels_with_n_coeffs_funcs, stbir__horizontal_gather_4_channels_with_n_coeffs_funcs, 0,0, stbir__horizontal_gather_7_channels_with_n_coeffs_funcs
6887+};
6888+
6889+static stbir__horizontal_gather_channels_func ** stbir__horizontal_gather_channels_funcs[8] =
6890+{
6891+ 0, stbir__horizontal_gather_1_channels_funcs, stbir__horizontal_gather_2_channels_funcs, stbir__horizontal_gather_3_channels_funcs, stbir__horizontal_gather_4_channels_funcs, 0,0, stbir__horizontal_gather_7_channels_funcs
6892+};
6893+
6894+// there are six resize classifications: 0 == vertical scatter, 1 == vertical gather < 1x scale, 2 == vertical gather 1x-2x scale, 4 == vertical gather < 3x scale, 4 == vertical gather > 3x scale, 5 == <=4 pixel height, 6 == <=4 pixel wide column
6895+#define STBIR_RESIZE_CLASSIFICATIONS 8
6896+
6897+static float stbir__compute_weights[5][STBIR_RESIZE_CLASSIFICATIONS][4]= // 5 = 0=1chan, 1=2chan, 2=3chan, 3=4chan, 4=7chan
6898+{
6899+ {
6900+ { 1.00000f, 1.00000f, 0.31250f, 1.00000f },
6901+ { 0.56250f, 0.59375f, 0.00000f, 0.96875f },
6902+ { 1.00000f, 0.06250f, 0.00000f, 1.00000f },
6903+ { 0.00000f, 0.09375f, 1.00000f, 1.00000f },
6904+ { 1.00000f, 1.00000f, 0.31250f, 1.00000f },
6905+ { 0.03125f, 0.12500f, 1.00000f, 1.00000f },
6906+ { 1.00000f, 1.00000f, 0.06250f, 1.00000f },
6907+ { 0.00000f, 1.00000f, 0.00000f, 0.03125f },
6908+ }, {
6909+ { 0.00000f, 0.84375f, 0.00000f, 0.03125f },
6910+ { 0.09375f, 0.93750f, 0.00000f, 0.78125f },
6911+ { 0.87500f, 0.21875f, 0.00000f, 0.96875f },
6912+ { 0.09375f, 0.09375f, 1.00000f, 1.00000f },
6913+ { 0.00000f, 0.84375f, 0.00000f, 0.03125f },
6914+ { 0.03125f, 0.12500f, 1.00000f, 1.00000f },
6915+ { 1.00000f, 1.00000f, 0.06250f, 1.00000f },
6916+ { 0.00000f, 1.00000f, 0.00000f, 0.53125f },
6917+ }, {
6918+ { 0.00000f, 0.53125f, 0.00000f, 0.03125f },
6919+ { 0.06250f, 0.96875f, 0.00000f, 0.53125f },
6920+ { 0.87500f, 0.18750f, 0.00000f, 0.93750f },
6921+ { 0.00000f, 0.09375f, 1.00000f, 1.00000f },
6922+ { 0.00000f, 0.53125f, 0.00000f, 0.03125f },
6923+ { 0.03125f, 0.12500f, 1.00000f, 1.00000f },
6924+ { 1.00000f, 1.00000f, 0.06250f, 1.00000f },
6925+ { 0.00000f, 1.00000f, 0.00000f, 0.56250f },
6926+ }, {
6927+ { 0.00000f, 0.50000f, 0.00000f, 0.71875f },
6928+ { 0.06250f, 0.84375f, 0.00000f, 0.87500f },
6929+ { 1.00000f, 0.50000f, 0.50000f, 0.96875f },
6930+ { 1.00000f, 0.09375f, 0.31250f, 0.50000f },
6931+ { 0.00000f, 0.50000f, 0.00000f, 0.71875f },
6932+ { 1.00000f, 0.03125f, 0.03125f, 0.53125f },
6933+ { 1.00000f, 1.00000f, 0.06250f, 1.00000f },
6934+ { 0.00000f, 1.00000f, 0.03125f, 0.18750f },
6935+ }, {
6936+ { 0.00000f, 0.59375f, 0.00000f, 0.96875f },
6937+ { 0.06250f, 0.81250f, 0.06250f, 0.59375f },
6938+ { 0.75000f, 0.43750f, 0.12500f, 0.96875f },
6939+ { 0.87500f, 0.06250f, 0.18750f, 0.43750f },
6940+ { 0.00000f, 0.59375f, 0.00000f, 0.96875f },
6941+ { 0.15625f, 0.12500f, 1.00000f, 1.00000f },
6942+ { 1.00000f, 1.00000f, 0.06250f, 1.00000f },
6943+ { 0.00000f, 1.00000f, 0.03125f, 0.34375f },
6944+ }
6945+};
6946+
6947+// structure that allow us to query and override info for training the costs
6948+typedef struct STBIR__V_FIRST_INFO
6949+{
6950+ double v_cost, h_cost;
6951+ int control_v_first; // 0 = no control, 1 = force hori, 2 = force vert
6952+ int v_first;
6953+ int v_resize_classification;
6954+ int is_gather;
6955+} STBIR__V_FIRST_INFO;
6956+
6957+#ifdef STBIR__V_FIRST_INFO_BUFFER
6958+static STBIR__V_FIRST_INFO STBIR__V_FIRST_INFO_BUFFER = {0};
6959+#define STBIR__V_FIRST_INFO_POINTER &STBIR__V_FIRST_INFO_BUFFER
6960+#else
6961+#define STBIR__V_FIRST_INFO_POINTER 0
6962+#endif
6963+
6964+// Figure out whether to scale along the horizontal or vertical first.
6965+// This only *super* important when you are scaling by a massively
6966+// different amount in the vertical vs the horizontal (for example, if
6967+// you are scaling by 2x in the width, and 0.5x in the height, then you
6968+// want to do the vertical scale first, because it's around 3x faster
6969+// in that order.
6970+//
6971+// In more normal circumstances, this makes a 20-40% differences, so
6972+// it's good to get right, but not critical. The normal way that you
6973+// decide which direction goes first is just figuring out which
6974+// direction does more multiplies. But with modern CPUs with their
6975+// fancy caches and SIMD and high IPC abilities, so there's just a lot
6976+// more that goes into it.
6977+//
6978+// My handwavy sort of solution is to have an app that does a whole
6979+// bunch of timing for both vertical and horizontal first modes,
6980+// and then another app that can read lots of these timing files
6981+// and try to search for the best weights to use. Dotimings.c
6982+// is the app that does a bunch of timings, and vf_train.c is the
6983+// app that solves for the best weights (and shows how well it
6984+// does currently).
6985+
6986+static int stbir__should_do_vertical_first( float weights_table[STBIR_RESIZE_CLASSIFICATIONS][4], int horizontal_filter_pixel_width, float horizontal_scale, int horizontal_output_size, int vertical_filter_pixel_width, float vertical_scale, int vertical_output_size, int is_gather, STBIR__V_FIRST_INFO * info )
6987+{
6988+ double v_cost, h_cost;
6989+ float * weights;
6990+ int vertical_first;
6991+ int v_classification;
6992+
6993+ // categorize the resize into buckets
6994+ if ( ( vertical_output_size <= 4 ) || ( horizontal_output_size <= 4 ) )
6995+ v_classification = ( vertical_output_size < horizontal_output_size ) ? 6 : 7;
6996+ else if ( ( !is_gather ) && ( ( vertical_output_size <= 16 ) || ( horizontal_output_size <= 16 ) ) )
6997+ v_classification = 4;
6998+ else if ( vertical_scale <= 1.0f )
6999+ v_classification = ( is_gather ) ? 1 : 0;
7000+ else if ( vertical_scale <= 2.0f)
7001+ v_classification = 2;
7002+ else if ( vertical_scale <= 3.0f)
7003+ v_classification = 3;
7004+ else
7005+ v_classification = 5; // everything bigger than 3x
7006+
7007+ // use the right weights
7008+ weights = weights_table[ v_classification ];
7009+
7010+ // this is the costs when you don't take into account modern CPUs with high ipc and simd and caches - wish we had a better estimate
7011+ h_cost = (float)horizontal_filter_pixel_width * weights[0] + horizontal_scale * (float)vertical_filter_pixel_width * weights[1];
7012+ v_cost = (float)vertical_filter_pixel_width * weights[2] + vertical_scale * (float)horizontal_filter_pixel_width * weights[3];
7013+
7014+ // use computation estimate to decide vertical first or not
7015+ vertical_first = ( v_cost <= h_cost ) ? 1 : 0;
7016+
7017+ // save these, if requested
7018+ if ( info )
7019+ {
7020+ info->h_cost = h_cost;
7021+ info->v_cost = v_cost;
7022+ info->v_resize_classification = v_classification;
7023+ info->v_first = vertical_first;
7024+ info->is_gather = is_gather;
7025+ }
7026+
7027+ // and this allows us to override everything for testing (see dotiming.c)
7028+ if ( ( info ) && ( info->control_v_first ) )
7029+ vertical_first = ( info->control_v_first == 2 ) ? 1 : 0;
7030+
7031+ return vertical_first;
7032+}
7033+
7034+// layout lookups - must match stbir_internal_pixel_layout
7035+static unsigned char stbir__pixel_channels[] = {
7036+ 1,2,3,3,4, // 1ch, 2ch, rgb, bgr, 4ch
7037+ 4,4,4,4,2,2, // RGBA,BGRA,ARGB,ABGR,RA,AR
7038+ 4,4,4,4,2,2, // RGBA_PM,BGRA_PM,ARGB_PM,ABGR_PM,RA_PM,AR_PM
7039+};
7040+
7041+// the internal pixel layout enums are in a different order, so we can easily do range comparisons of types
7042+// the public pixel layout is ordered in a way that if you cast num_channels (1-4) to the enum, you get something sensible
7043+static stbir_internal_pixel_layout stbir__pixel_layout_convert_public_to_internal[] = {
7044+ STBIRI_BGR, STBIRI_1CHANNEL, STBIRI_2CHANNEL, STBIRI_RGB, STBIRI_RGBA,
7045+ STBIRI_4CHANNEL, STBIRI_BGRA, STBIRI_ARGB, STBIRI_ABGR, STBIRI_RA, STBIRI_AR,
7046+ STBIRI_RGBA_PM, STBIRI_BGRA_PM, STBIRI_ARGB_PM, STBIRI_ABGR_PM, STBIRI_RA_PM, STBIRI_AR_PM,
7047+};
7048+
7049+static stbir__info * stbir__alloc_internal_mem_and_build_samplers( stbir__sampler * horizontal, stbir__sampler * vertical, stbir__contributors * conservative, stbir_pixel_layout input_pixel_layout_public, stbir_pixel_layout output_pixel_layout_public, int splits, int new_x, int new_y, int fast_alpha, void * user_data STBIR_ONLY_PROFILE_BUILD_GET_INFO )
7050+{
7051+ static char stbir_channel_count_index[8]={ 9,0,1,2, 3,9,9,4 };
7052+
7053+ stbir__info * info = 0;
7054+ void * alloced = 0;
7055+ size_t alloced_total = 0;
7056+ int vertical_first;
7057+ size_t decode_buffer_size, ring_buffer_length_bytes, ring_buffer_size, vertical_buffer_size;
7058+ int alloc_ring_buffer_num_entries;
7059+
7060+ int alpha_weighting_type = 0; // 0=none, 1=simple, 2=fancy
7061+ int conservative_split_output_size = stbir__get_max_split( splits, vertical->scale_info.output_sub_size );
7062+ stbir_internal_pixel_layout input_pixel_layout = stbir__pixel_layout_convert_public_to_internal[ input_pixel_layout_public ];
7063+ stbir_internal_pixel_layout output_pixel_layout = stbir__pixel_layout_convert_public_to_internal[ output_pixel_layout_public ];
7064+ int channels = stbir__pixel_channels[ input_pixel_layout ];
7065+ int effective_channels = channels;
7066+
7067+ // first figure out what type of alpha weighting to use (if any)
7068+ if ( ( horizontal->filter_enum != STBIR_FILTER_POINT_SAMPLE ) || ( vertical->filter_enum != STBIR_FILTER_POINT_SAMPLE ) ) // no alpha weighting on point sampling
7069+ {
7070+ if ( ( input_pixel_layout >= STBIRI_RGBA ) && ( input_pixel_layout <= STBIRI_AR ) && ( output_pixel_layout >= STBIRI_RGBA ) && ( output_pixel_layout <= STBIRI_AR ) )
7071+ {
7072+ if ( fast_alpha )
7073+ {
7074+ alpha_weighting_type = 4;
7075+ }
7076+ else
7077+ {
7078+ static int fancy_alpha_effective_cnts[6] = { 7, 7, 7, 7, 3, 3 };
7079+ alpha_weighting_type = 2;
7080+ effective_channels = fancy_alpha_effective_cnts[ input_pixel_layout - STBIRI_RGBA ];
7081+ }
7082+ }
7083+ else if ( ( input_pixel_layout >= STBIRI_RGBA_PM ) && ( input_pixel_layout <= STBIRI_AR_PM ) && ( output_pixel_layout >= STBIRI_RGBA ) && ( output_pixel_layout <= STBIRI_AR ) )
7084+ {
7085+ // input premult, output non-premult
7086+ alpha_weighting_type = 3;
7087+ }
7088+ else if ( ( input_pixel_layout >= STBIRI_RGBA ) && ( input_pixel_layout <= STBIRI_AR ) && ( output_pixel_layout >= STBIRI_RGBA_PM ) && ( output_pixel_layout <= STBIRI_AR_PM ) )
7089+ {
7090+ // input non-premult, output premult
7091+ alpha_weighting_type = 1;
7092+ }
7093+ }
7094+
7095+ // channel in and out count must match currently
7096+ if ( channels != stbir__pixel_channels[ output_pixel_layout ] )
7097+ return 0;
7098+
7099+ // get vertical first
7100+ vertical_first = stbir__should_do_vertical_first( stbir__compute_weights[ (int)stbir_channel_count_index[ effective_channels ] ], horizontal->filter_pixel_width, horizontal->scale_info.scale, horizontal->scale_info.output_sub_size, vertical->filter_pixel_width, vertical->scale_info.scale, vertical->scale_info.output_sub_size, vertical->is_gather, STBIR__V_FIRST_INFO_POINTER );
7101+
7102+ // sometimes read one float off in some of the unrolled loops (with a weight of zero coeff, so it doesn't have an effect)
7103+ // we use a few extra floats instead of just 1, so that input callback buffer can overlap with the decode buffer without
7104+ // the conversion routines overwriting the callback input data.
7105+ decode_buffer_size = ( conservative->n1 - conservative->n0 + 1 ) * effective_channels * sizeof(float) + sizeof(float)*STBIR_INPUT_CALLBACK_PADDING; // extra floats for input callback stagger
7106+
7107+#if defined( STBIR__SEPARATE_ALLOCATIONS ) && defined(STBIR_SIMD8)
7108+ if ( effective_channels == 3 )
7109+ decode_buffer_size += sizeof(float); // avx in 3 channel mode needs one float at the start of the buffer (only with separate allocations)
7110+#endif
7111+
7112+ ring_buffer_length_bytes = (size_t)horizontal->scale_info.output_sub_size * (size_t)effective_channels * sizeof(float) + sizeof(float)*STBIR_INPUT_CALLBACK_PADDING; // extra floats for padding
7113+
7114+ // if we do vertical first, the ring buffer holds a whole decoded line
7115+ if ( vertical_first )
7116+ ring_buffer_length_bytes = ( decode_buffer_size + 15 ) & ~15;
7117+
7118+ if ( ( ring_buffer_length_bytes & 4095 ) == 0 ) ring_buffer_length_bytes += 64*3; // avoid 4k alias
7119+
7120+ // One extra entry because floating point precision problems sometimes cause an extra to be necessary.
7121+ alloc_ring_buffer_num_entries = vertical->filter_pixel_width + 1;
7122+
7123+ // we never need more ring buffer entries than the scanlines we're outputting when in scatter mode
7124+ if ( ( !vertical->is_gather ) && ( alloc_ring_buffer_num_entries > conservative_split_output_size ) )
7125+ alloc_ring_buffer_num_entries = conservative_split_output_size;
7126+
7127+ ring_buffer_size = (size_t)alloc_ring_buffer_num_entries * (size_t)ring_buffer_length_bytes;
7128+
7129+ // The vertical buffer is used differently, depending on whether we are scattering
7130+ // the vertical scanlines, or gathering them.
7131+ // If scattering, it's used at the temp buffer to accumulate each output.
7132+ // If gathering, it's just the output buffer.
7133+ vertical_buffer_size = (size_t)horizontal->scale_info.output_sub_size * (size_t)effective_channels * sizeof(float) + sizeof(float); // extra float for padding
7134+
7135+ // we make two passes through this loop, 1st to add everything up, 2nd to allocate and init
7136+ for(;;)
7137+ {
7138+ int i;
7139+ void * advance_mem = alloced;
7140+ int copy_horizontal = 0;
7141+ stbir__sampler * possibly_use_horizontal_for_pivot = 0;
7142+
7143+#ifdef STBIR__SEPARATE_ALLOCATIONS
7144+ #define STBIR__NEXT_PTR( ptr, size, ntype ) if ( alloced ) { void * p = STBIR_MALLOC( size, user_data); if ( p == 0 ) { stbir__free_internal_mem( info ); return 0; } (ptr) = (ntype*)p; }
7145+#else
7146+ #define STBIR__NEXT_PTR( ptr, size, ntype ) advance_mem = (void*) ( ( ((size_t)advance_mem) + 15 ) & ~15 ); if ( alloced ) ptr = (ntype*)advance_mem; advance_mem = (char*)(((size_t)advance_mem) + (size));
7147+#endif
7148+
7149+ STBIR__NEXT_PTR( info, sizeof( stbir__info ), stbir__info );
7150+
7151+ STBIR__NEXT_PTR( info->split_info, sizeof( stbir__per_split_info ) * splits, stbir__per_split_info );
7152+
7153+ if ( info )
7154+ {
7155+ static stbir__alpha_weight_func * fancy_alpha_weights[6] = { stbir__fancy_alpha_weight_4ch, stbir__fancy_alpha_weight_4ch, stbir__fancy_alpha_weight_4ch, stbir__fancy_alpha_weight_4ch, stbir__fancy_alpha_weight_2ch, stbir__fancy_alpha_weight_2ch };
7156+ static stbir__alpha_unweight_func * fancy_alpha_unweights[6] = { stbir__fancy_alpha_unweight_4ch, stbir__fancy_alpha_unweight_4ch, stbir__fancy_alpha_unweight_4ch, stbir__fancy_alpha_unweight_4ch, stbir__fancy_alpha_unweight_2ch, stbir__fancy_alpha_unweight_2ch };
7157+ static stbir__alpha_weight_func * simple_alpha_weights[6] = { stbir__simple_alpha_weight_4ch, stbir__simple_alpha_weight_4ch, stbir__simple_alpha_weight_4ch, stbir__simple_alpha_weight_4ch, stbir__simple_alpha_weight_2ch, stbir__simple_alpha_weight_2ch };
7158+ static stbir__alpha_unweight_func * simple_alpha_unweights[6] = { stbir__simple_alpha_unweight_4ch, stbir__simple_alpha_unweight_4ch, stbir__simple_alpha_unweight_4ch, stbir__simple_alpha_unweight_4ch, stbir__simple_alpha_unweight_2ch, stbir__simple_alpha_unweight_2ch };
7159+
7160+ // initialize info fields
7161+ info->alloced_mem = alloced;
7162+ info->alloced_total = alloced_total;
7163+
7164+ info->channels = channels;
7165+ info->effective_channels = effective_channels;
7166+
7167+ info->offset_x = new_x;
7168+ info->offset_y = new_y;
7169+ info->alloc_ring_buffer_num_entries = (int)alloc_ring_buffer_num_entries;
7170+ info->ring_buffer_num_entries = 0;
7171+ info->ring_buffer_length_bytes = (int)ring_buffer_length_bytes;
7172+ info->splits = splits;
7173+ info->vertical_first = vertical_first;
7174+
7175+ info->input_pixel_layout_internal = input_pixel_layout;
7176+ info->output_pixel_layout_internal = output_pixel_layout;
7177+
7178+ // setup alpha weight functions
7179+ info->alpha_weight = 0;
7180+ info->alpha_unweight = 0;
7181+
7182+ // handle alpha weighting functions and overrides
7183+ if ( alpha_weighting_type == 2 )
7184+ {
7185+ // high quality alpha multiplying on the way in, dividing on the way out
7186+ info->alpha_weight = fancy_alpha_weights[ input_pixel_layout - STBIRI_RGBA ];
7187+ info->alpha_unweight = fancy_alpha_unweights[ output_pixel_layout - STBIRI_RGBA ];
7188+ }
7189+ else if ( alpha_weighting_type == 4 )
7190+ {
7191+ // fast alpha multiplying on the way in, dividing on the way out
7192+ info->alpha_weight = simple_alpha_weights[ input_pixel_layout - STBIRI_RGBA ];
7193+ info->alpha_unweight = simple_alpha_unweights[ output_pixel_layout - STBIRI_RGBA ];
7194+ }
7195+ else if ( alpha_weighting_type == 1 )
7196+ {
7197+ // fast alpha on the way in, leave in premultiplied form on way out
7198+ info->alpha_weight = simple_alpha_weights[ input_pixel_layout - STBIRI_RGBA ];
7199+ }
7200+ else if ( alpha_weighting_type == 3 )
7201+ {
7202+ // incoming is premultiplied, fast alpha dividing on the way out - non-premultiplied output
7203+ info->alpha_unweight = simple_alpha_unweights[ output_pixel_layout - STBIRI_RGBA ];
7204+ }
7205+
7206+ // handle 3-chan color flipping, using the alpha weight path
7207+ if ( ( ( input_pixel_layout == STBIRI_RGB ) && ( output_pixel_layout == STBIRI_BGR ) ) ||
7208+ ( ( input_pixel_layout == STBIRI_BGR ) && ( output_pixel_layout == STBIRI_RGB ) ) )
7209+ {
7210+ // do the flipping on the smaller of the two ends
7211+ if ( horizontal->scale_info.scale < 1.0f )
7212+ info->alpha_unweight = stbir__simple_flip_3ch;
7213+ else
7214+ info->alpha_weight = stbir__simple_flip_3ch;
7215+ }
7216+
7217+ }
7218+
7219+ // get all the per-split buffers
7220+ for( i = 0 ; i < splits ; i++ )
7221+ {
7222+ STBIR__NEXT_PTR( info->split_info[i].decode_buffer, decode_buffer_size, float );
7223+
7224+#ifdef STBIR__SEPARATE_ALLOCATIONS
7225+
7226+ #ifdef STBIR_SIMD8
7227+ if ( ( info ) && ( effective_channels == 3 ) )
7228+ ++info->split_info[i].decode_buffer; // avx in 3 channel mode needs one float at the start of the buffer
7229+ #endif
7230+
7231+ STBIR__NEXT_PTR( info->split_info[i].ring_buffers, alloc_ring_buffer_num_entries * sizeof(float*), float* );
7232+ {
7233+ int j;
7234+ for( j = 0 ; j < alloc_ring_buffer_num_entries ; j++ )
7235+ {
7236+ STBIR__NEXT_PTR( info->split_info[i].ring_buffers[j], ring_buffer_length_bytes, float );
7237+ #ifdef STBIR_SIMD8
7238+ if ( ( info ) && ( effective_channels == 3 ) )
7239+ ++info->split_info[i].ring_buffers[j]; // avx in 3 channel mode needs one float at the start of the buffer
7240+ #endif
7241+ }
7242+ }
7243+#else
7244+ STBIR__NEXT_PTR( info->split_info[i].ring_buffer, ring_buffer_size, float );
7245+#endif
7246+ STBIR__NEXT_PTR( info->split_info[i].vertical_buffer, vertical_buffer_size, float );
7247+ }
7248+
7249+ // alloc memory for to-be-pivoted coeffs (if necessary)
7250+ if ( vertical->is_gather == 0 )
7251+ {
7252+ size_t both;
7253+ size_t temp_mem_amt;
7254+
7255+ // when in vertical scatter mode, we first build the coefficients in gather mode, and then pivot after,
7256+ // that means we need two buffers, so we try to use the decode buffer and ring buffer for this. if that
7257+ // is too small, we just allocate extra memory to use as this temp.
7258+
7259+ both = (size_t)vertical->gather_prescatter_contributors_size + (size_t)vertical->gather_prescatter_coefficients_size;
7260+
7261+#ifdef STBIR__SEPARATE_ALLOCATIONS
7262+ temp_mem_amt = decode_buffer_size;
7263+
7264+ #ifdef STBIR_SIMD8
7265+ if ( effective_channels == 3 )
7266+ --temp_mem_amt; // avx in 3 channel mode needs one float at the start of the buffer
7267+ #endif
7268+#else
7269+ temp_mem_amt = (size_t)( decode_buffer_size + ring_buffer_size + vertical_buffer_size ) * (size_t)splits;
7270+#endif
7271+ if ( temp_mem_amt >= both )
7272+ {
7273+ if ( info )
7274+ {
7275+ vertical->gather_prescatter_contributors = (stbir__contributors*)info->split_info[0].decode_buffer;
7276+ vertical->gather_prescatter_coefficients = (float*) ( ( (char*)info->split_info[0].decode_buffer ) + vertical->gather_prescatter_contributors_size );
7277+ }
7278+ }
7279+ else
7280+ {
7281+ // ring+decode memory is too small, so allocate temp memory
7282+ STBIR__NEXT_PTR( vertical->gather_prescatter_contributors, vertical->gather_prescatter_contributors_size, stbir__contributors );
7283+ STBIR__NEXT_PTR( vertical->gather_prescatter_coefficients, vertical->gather_prescatter_coefficients_size, float );
7284+ }
7285+ }
7286+
7287+ STBIR__NEXT_PTR( horizontal->contributors, horizontal->contributors_size, stbir__contributors );
7288+ STBIR__NEXT_PTR( horizontal->coefficients, horizontal->coefficients_size, float );
7289+
7290+ // are the two filters identical?? (happens a lot with mipmap generation)
7291+ if ( ( horizontal->filter_kernel == vertical->filter_kernel ) && ( horizontal->filter_support == vertical->filter_support ) && ( horizontal->edge == vertical->edge ) && ( horizontal->scale_info.output_sub_size == vertical->scale_info.output_sub_size ) )
7292+ {
7293+ float diff_scale = horizontal->scale_info.scale - vertical->scale_info.scale;
7294+ float diff_shift = horizontal->scale_info.pixel_shift - vertical->scale_info.pixel_shift;
7295+ if ( diff_scale < 0.0f ) diff_scale = -diff_scale;
7296+ if ( diff_shift < 0.0f ) diff_shift = -diff_shift;
7297+ if ( ( diff_scale <= stbir__small_float ) && ( diff_shift <= stbir__small_float ) )
7298+ {
7299+ if ( horizontal->is_gather == vertical->is_gather )
7300+ {
7301+ copy_horizontal = 1;
7302+ goto no_vert_alloc;
7303+ }
7304+ // everything matches, but vertical is scatter, horizontal is gather, use horizontal coeffs for vertical pivot coeffs
7305+ possibly_use_horizontal_for_pivot = horizontal;
7306+ }
7307+ }
7308+
7309+ STBIR__NEXT_PTR( vertical->contributors, vertical->contributors_size, stbir__contributors );
7310+ STBIR__NEXT_PTR( vertical->coefficients, vertical->coefficients_size, float );
7311+
7312+ no_vert_alloc:
7313+
7314+ if ( info )
7315+ {
7316+ STBIR_PROFILE_BUILD_START( horizontal );
7317+
7318+ stbir__calculate_filters( horizontal, 0, user_data STBIR_ONLY_PROFILE_BUILD_SET_INFO );
7319+
7320+ // setup the horizontal gather functions
7321+ // start with defaulting to the n_coeffs functions (specialized on channels and remnant leftover)
7322+ info->horizontal_gather_channels = stbir__horizontal_gather_n_coeffs_funcs[ effective_channels ][ horizontal->extent_info.widest & 3 ];
7323+ // but if the number of coeffs <= 12, use another set of special cases. <=12 coeffs is any enlarging resize, or shrinking resize down to about 1/3 size
7324+ if ( horizontal->extent_info.widest <= 12 )
7325+ info->horizontal_gather_channels = stbir__horizontal_gather_channels_funcs[ effective_channels ][ horizontal->extent_info.widest - 1 ];
7326+
7327+ info->scanline_extents.conservative.n0 = conservative->n0;
7328+ info->scanline_extents.conservative.n1 = conservative->n1;
7329+
7330+ // get exact extents
7331+ stbir__get_extents( horizontal, &info->scanline_extents );
7332+
7333+ // pack the horizontal coeffs
7334+ horizontal->coefficient_width = stbir__pack_coefficients(horizontal->num_contributors, horizontal->contributors, horizontal->coefficients, horizontal->coefficient_width, horizontal->extent_info.widest, info->scanline_extents.conservative.n0, info->scanline_extents.conservative.n1 );
7335+
7336+ STBIR_MEMCPY( &info->horizontal, horizontal, sizeof( stbir__sampler ) );
7337+
7338+ STBIR_PROFILE_BUILD_END( horizontal );
7339+
7340+ if ( copy_horizontal )
7341+ {
7342+ STBIR_MEMCPY( &info->vertical, horizontal, sizeof( stbir__sampler ) );
7343+ }
7344+ else
7345+ {
7346+ STBIR_PROFILE_BUILD_START( vertical );
7347+
7348+ stbir__calculate_filters( vertical, possibly_use_horizontal_for_pivot, user_data STBIR_ONLY_PROFILE_BUILD_SET_INFO );
7349+ STBIR_MEMCPY( &info->vertical, vertical, sizeof( stbir__sampler ) );
7350+
7351+ STBIR_PROFILE_BUILD_END( vertical );
7352+ }
7353+
7354+ // setup the vertical split ranges
7355+ stbir__get_split_info( info->split_info, info->splits, info->vertical.scale_info.output_sub_size, info->vertical.filter_pixel_margin, info->vertical.scale_info.input_full_size, info->vertical.is_gather, info->vertical.contributors );
7356+
7357+ // now we know precisely how many entries we need
7358+ info->ring_buffer_num_entries = info->vertical.extent_info.widest;
7359+
7360+ // we never need more ring buffer entries than the scanlines we're outputting
7361+ if ( ( !info->vertical.is_gather ) && ( info->ring_buffer_num_entries > conservative_split_output_size ) )
7362+ info->ring_buffer_num_entries = conservative_split_output_size;
7363+ STBIR_ASSERT( info->ring_buffer_num_entries <= info->alloc_ring_buffer_num_entries );
7364+ }
7365+ #undef STBIR__NEXT_PTR
7366+
7367+
7368+ // is this the first time through loop?
7369+ if ( info == 0 )
7370+ {
7371+ alloced_total = ( 15 + (size_t)advance_mem );
7372+ alloced = STBIR_MALLOC( alloced_total, user_data );
7373+ if ( alloced == 0 )
7374+ return 0;
7375+ }
7376+ else
7377+ return info; // success
7378+ }
7379+}
7380+
7381+static int stbir__perform_resize( stbir__info const * info, int split_start, int split_count )
7382+{
7383+ stbir__per_split_info * split_info = info->split_info + split_start;
7384+
7385+ STBIR_PROFILE_CLEAR_EXTRAS();
7386+
7387+ STBIR_PROFILE_FIRST_START( looping );
7388+ if (info->vertical.is_gather)
7389+ stbir__vertical_gather_loop( info, split_info, split_count );
7390+ else
7391+ stbir__vertical_scatter_loop( info, split_info, split_count );
7392+ STBIR_PROFILE_END( looping );
7393+
7394+ return 1;
7395+}
7396+
7397+static void stbir__update_info_from_resize( stbir__info * info, STBIR_RESIZE * resize )
7398+{
7399+ static stbir__decode_pixels_func * decode_simple[STBIR_TYPE_HALF_FLOAT-STBIR_TYPE_UINT8_SRGB+1]=
7400+ {
7401+ /* 1ch-4ch */ stbir__decode_uint8_srgb, stbir__decode_uint8_srgb, 0, stbir__decode_float_linear, stbir__decode_half_float_linear,
7402+ };
7403+
7404+ static stbir__decode_pixels_func * decode_alphas[STBIRI_AR-STBIRI_RGBA+1][STBIR_TYPE_HALF_FLOAT-STBIR_TYPE_UINT8_SRGB+1]=
7405+ {
7406+ { /* RGBA */ stbir__decode_uint8_srgb4_linearalpha, stbir__decode_uint8_srgb, 0, stbir__decode_float_linear, stbir__decode_half_float_linear },
7407+ { /* BGRA */ stbir__decode_uint8_srgb4_linearalpha_BGRA, stbir__decode_uint8_srgb_BGRA, 0, stbir__decode_float_linear_BGRA, stbir__decode_half_float_linear_BGRA },
7408+ { /* ARGB */ stbir__decode_uint8_srgb4_linearalpha_ARGB, stbir__decode_uint8_srgb_ARGB, 0, stbir__decode_float_linear_ARGB, stbir__decode_half_float_linear_ARGB },
7409+ { /* ABGR */ stbir__decode_uint8_srgb4_linearalpha_ABGR, stbir__decode_uint8_srgb_ABGR, 0, stbir__decode_float_linear_ABGR, stbir__decode_half_float_linear_ABGR },
7410+ { /* RA */ stbir__decode_uint8_srgb2_linearalpha, stbir__decode_uint8_srgb, 0, stbir__decode_float_linear, stbir__decode_half_float_linear },
7411+ { /* AR */ stbir__decode_uint8_srgb2_linearalpha_AR, stbir__decode_uint8_srgb_AR, 0, stbir__decode_float_linear_AR, stbir__decode_half_float_linear_AR },
7412+ };
7413+
7414+ static stbir__decode_pixels_func * decode_simple_scaled_or_not[2][2]=
7415+ {
7416+ { stbir__decode_uint8_linear_scaled, stbir__decode_uint8_linear }, { stbir__decode_uint16_linear_scaled, stbir__decode_uint16_linear },
7417+ };
7418+
7419+ static stbir__decode_pixels_func * decode_alphas_scaled_or_not[STBIRI_AR-STBIRI_RGBA+1][2][2]=
7420+ {
7421+ { /* RGBA */ { stbir__decode_uint8_linear_scaled, stbir__decode_uint8_linear }, { stbir__decode_uint16_linear_scaled, stbir__decode_uint16_linear } },
7422+ { /* BGRA */ { stbir__decode_uint8_linear_scaled_BGRA, stbir__decode_uint8_linear_BGRA }, { stbir__decode_uint16_linear_scaled_BGRA, stbir__decode_uint16_linear_BGRA } },
7423+ { /* ARGB */ { stbir__decode_uint8_linear_scaled_ARGB, stbir__decode_uint8_linear_ARGB }, { stbir__decode_uint16_linear_scaled_ARGB, stbir__decode_uint16_linear_ARGB } },
7424+ { /* ABGR */ { stbir__decode_uint8_linear_scaled_ABGR, stbir__decode_uint8_linear_ABGR }, { stbir__decode_uint16_linear_scaled_ABGR, stbir__decode_uint16_linear_ABGR } },
7425+ { /* RA */ { stbir__decode_uint8_linear_scaled, stbir__decode_uint8_linear }, { stbir__decode_uint16_linear_scaled, stbir__decode_uint16_linear } },
7426+ { /* AR */ { stbir__decode_uint8_linear_scaled_AR, stbir__decode_uint8_linear_AR }, { stbir__decode_uint16_linear_scaled_AR, stbir__decode_uint16_linear_AR } }
7427+ };
7428+
7429+ static stbir__encode_pixels_func * encode_simple[STBIR_TYPE_HALF_FLOAT-STBIR_TYPE_UINT8_SRGB+1]=
7430+ {
7431+ /* 1ch-4ch */ stbir__encode_uint8_srgb, stbir__encode_uint8_srgb, 0, stbir__encode_float_linear, stbir__encode_half_float_linear,
7432+ };
7433+
7434+ static stbir__encode_pixels_func * encode_alphas[STBIRI_AR-STBIRI_RGBA+1][STBIR_TYPE_HALF_FLOAT-STBIR_TYPE_UINT8_SRGB+1]=
7435+ {
7436+ { /* RGBA */ stbir__encode_uint8_srgb4_linearalpha, stbir__encode_uint8_srgb, 0, stbir__encode_float_linear, stbir__encode_half_float_linear },
7437+ { /* BGRA */ stbir__encode_uint8_srgb4_linearalpha_BGRA, stbir__encode_uint8_srgb_BGRA, 0, stbir__encode_float_linear_BGRA, stbir__encode_half_float_linear_BGRA },
7438+ { /* ARGB */ stbir__encode_uint8_srgb4_linearalpha_ARGB, stbir__encode_uint8_srgb_ARGB, 0, stbir__encode_float_linear_ARGB, stbir__encode_half_float_linear_ARGB },
7439+ { /* ABGR */ stbir__encode_uint8_srgb4_linearalpha_ABGR, stbir__encode_uint8_srgb_ABGR, 0, stbir__encode_float_linear_ABGR, stbir__encode_half_float_linear_ABGR },
7440+ { /* RA */ stbir__encode_uint8_srgb2_linearalpha, stbir__encode_uint8_srgb, 0, stbir__encode_float_linear, stbir__encode_half_float_linear },
7441+ { /* AR */ stbir__encode_uint8_srgb2_linearalpha_AR, stbir__encode_uint8_srgb_AR, 0, stbir__encode_float_linear_AR, stbir__encode_half_float_linear_AR }
7442+ };
7443+
7444+ static stbir__encode_pixels_func * encode_simple_scaled_or_not[2][2]=
7445+ {
7446+ { stbir__encode_uint8_linear_scaled, stbir__encode_uint8_linear }, { stbir__encode_uint16_linear_scaled, stbir__encode_uint16_linear },
7447+ };
7448+
7449+ static stbir__encode_pixels_func * encode_alphas_scaled_or_not[STBIRI_AR-STBIRI_RGBA+1][2][2]=
7450+ {
7451+ { /* RGBA */ { stbir__encode_uint8_linear_scaled, stbir__encode_uint8_linear }, { stbir__encode_uint16_linear_scaled, stbir__encode_uint16_linear } },
7452+ { /* BGRA */ { stbir__encode_uint8_linear_scaled_BGRA, stbir__encode_uint8_linear_BGRA }, { stbir__encode_uint16_linear_scaled_BGRA, stbir__encode_uint16_linear_BGRA } },
7453+ { /* ARGB */ { stbir__encode_uint8_linear_scaled_ARGB, stbir__encode_uint8_linear_ARGB }, { stbir__encode_uint16_linear_scaled_ARGB, stbir__encode_uint16_linear_ARGB } },
7454+ { /* ABGR */ { stbir__encode_uint8_linear_scaled_ABGR, stbir__encode_uint8_linear_ABGR }, { stbir__encode_uint16_linear_scaled_ABGR, stbir__encode_uint16_linear_ABGR } },
7455+ { /* RA */ { stbir__encode_uint8_linear_scaled, stbir__encode_uint8_linear }, { stbir__encode_uint16_linear_scaled, stbir__encode_uint16_linear } },
7456+ { /* AR */ { stbir__encode_uint8_linear_scaled_AR, stbir__encode_uint8_linear_AR }, { stbir__encode_uint16_linear_scaled_AR, stbir__encode_uint16_linear_AR } }
7457+ };
7458+
7459+ stbir__decode_pixels_func * decode_pixels = 0;
7460+ stbir__encode_pixels_func * encode_pixels = 0;
7461+ stbir_datatype input_type, output_type;
7462+
7463+ input_type = resize->input_data_type;
7464+ output_type = resize->output_data_type;
7465+ info->input_data = resize->input_pixels;
7466+ info->input_stride_bytes = resize->input_stride_in_bytes;
7467+ info->output_stride_bytes = resize->output_stride_in_bytes;
7468+
7469+ // if we're completely point sampling, then we can turn off SRGB
7470+ if ( ( info->horizontal.filter_enum == STBIR_FILTER_POINT_SAMPLE ) && ( info->vertical.filter_enum == STBIR_FILTER_POINT_SAMPLE ) )
7471+ {
7472+ if ( ( ( input_type == STBIR_TYPE_UINT8_SRGB ) || ( input_type == STBIR_TYPE_UINT8_SRGB_ALPHA ) ) &&
7473+ ( ( output_type == STBIR_TYPE_UINT8_SRGB ) || ( output_type == STBIR_TYPE_UINT8_SRGB_ALPHA ) ) )
7474+ {
7475+ input_type = STBIR_TYPE_UINT8;
7476+ output_type = STBIR_TYPE_UINT8;
7477+ }
7478+ }
7479+
7480+ // recalc the output and input strides
7481+ if ( info->input_stride_bytes == 0 )
7482+ info->input_stride_bytes = info->channels * info->horizontal.scale_info.input_full_size * stbir__type_size[input_type];
7483+
7484+ if ( info->output_stride_bytes == 0 )
7485+ info->output_stride_bytes = info->channels * info->horizontal.scale_info.output_sub_size * stbir__type_size[output_type];
7486+
7487+ // calc offset
7488+ info->output_data = ( (char*) resize->output_pixels ) + ( (size_t) info->offset_y * (size_t) resize->output_stride_in_bytes ) + ( info->offset_x * info->channels * stbir__type_size[output_type] );
7489+
7490+ info->in_pixels_cb = resize->input_cb;
7491+ info->user_data = resize->user_data;
7492+ info->out_pixels_cb = resize->output_cb;
7493+
7494+ // setup the input format converters
7495+ if ( ( input_type == STBIR_TYPE_UINT8 ) || ( input_type == STBIR_TYPE_UINT16 ) )
7496+ {
7497+ int non_scaled = 0;
7498+
7499+ // check if we can run unscaled - 0-255.0/0-65535.0 instead of 0-1.0 (which is a tiny bit faster when doing linear 8->8 or 16->16)
7500+ if ( ( !info->alpha_weight ) && ( !info->alpha_unweight ) ) // don't short circuit when alpha weighting (get everything to 0-1.0 as usual)
7501+ if ( ( ( input_type == STBIR_TYPE_UINT8 ) && ( output_type == STBIR_TYPE_UINT8 ) ) || ( ( input_type == STBIR_TYPE_UINT16 ) && ( output_type == STBIR_TYPE_UINT16 ) ) )
7502+ non_scaled = 1;
7503+
7504+ if ( info->input_pixel_layout_internal <= STBIRI_4CHANNEL )
7505+ decode_pixels = decode_simple_scaled_or_not[ input_type == STBIR_TYPE_UINT16 ][ non_scaled ];
7506+ else
7507+ decode_pixels = decode_alphas_scaled_or_not[ ( info->input_pixel_layout_internal - STBIRI_RGBA ) % ( STBIRI_AR-STBIRI_RGBA+1 ) ][ input_type == STBIR_TYPE_UINT16 ][ non_scaled ];
7508+ }
7509+ else
7510+ {
7511+ if ( info->input_pixel_layout_internal <= STBIRI_4CHANNEL )
7512+ decode_pixels = decode_simple[ input_type - STBIR_TYPE_UINT8_SRGB ];
7513+ else
7514+ decode_pixels = decode_alphas[ ( info->input_pixel_layout_internal - STBIRI_RGBA ) % ( STBIRI_AR-STBIRI_RGBA+1 ) ][ input_type - STBIR_TYPE_UINT8_SRGB ];
7515+ }
7516+
7517+ // setup the output format converters
7518+ if ( ( output_type == STBIR_TYPE_UINT8 ) || ( output_type == STBIR_TYPE_UINT16 ) )
7519+ {
7520+ int non_scaled = 0;
7521+
7522+ // check if we can run unscaled - 0-255.0/0-65535.0 instead of 0-1.0 (which is a tiny bit faster when doing linear 8->8 or 16->16)
7523+ if ( ( !info->alpha_weight ) && ( !info->alpha_unweight ) ) // don't short circuit when alpha weighting (get everything to 0-1.0 as usual)
7524+ if ( ( ( input_type == STBIR_TYPE_UINT8 ) && ( output_type == STBIR_TYPE_UINT8 ) ) || ( ( input_type == STBIR_TYPE_UINT16 ) && ( output_type == STBIR_TYPE_UINT16 ) ) )
7525+ non_scaled = 1;
7526+
7527+ if ( info->output_pixel_layout_internal <= STBIRI_4CHANNEL )
7528+ encode_pixels = encode_simple_scaled_or_not[ output_type == STBIR_TYPE_UINT16 ][ non_scaled ];
7529+ else
7530+ encode_pixels = encode_alphas_scaled_or_not[ ( info->output_pixel_layout_internal - STBIRI_RGBA ) % ( STBIRI_AR-STBIRI_RGBA+1 ) ][ output_type == STBIR_TYPE_UINT16 ][ non_scaled ];
7531+ }
7532+ else
7533+ {
7534+ if ( info->output_pixel_layout_internal <= STBIRI_4CHANNEL )
7535+ encode_pixels = encode_simple[ output_type - STBIR_TYPE_UINT8_SRGB ];
7536+ else
7537+ encode_pixels = encode_alphas[ ( info->output_pixel_layout_internal - STBIRI_RGBA ) % ( STBIRI_AR-STBIRI_RGBA+1 ) ][ output_type - STBIR_TYPE_UINT8_SRGB ];
7538+ }
7539+
7540+ info->input_type = input_type;
7541+ info->output_type = output_type;
7542+ info->decode_pixels = decode_pixels;
7543+ info->encode_pixels = encode_pixels;
7544+}
7545+
7546+static void stbir__clip( int * outx, int * outsubw, int outw, double * u0, double * u1 )
7547+{
7548+ double per, adj;
7549+ int over;
7550+
7551+ // do left/top edge
7552+ if ( *outx < 0 )
7553+ {
7554+ per = ( (double)*outx ) / ( (double)*outsubw ); // is negative
7555+ adj = per * ( *u1 - *u0 );
7556+ *u0 -= adj; // increases u0
7557+ *outx = 0;
7558+ }
7559+
7560+ // do right/bot edge
7561+ over = outw - ( *outx + *outsubw );
7562+ if ( over < 0 )
7563+ {
7564+ per = ( (double)over ) / ( (double)*outsubw ); // is negative
7565+ adj = per * ( *u1 - *u0 );
7566+ *u1 += adj; // decrease u1
7567+ *outsubw = outw - *outx;
7568+ }
7569+}
7570+
7571+// converts a double to a rational that has less than one float bit of error (returns 0 if unable to do so)
7572+static int stbir__double_to_rational(double f, stbir_uint32 limit, stbir_uint32 *numer, stbir_uint32 *denom, int limit_denom ) // limit_denom (1) or limit numer (0)
7573+{
7574+ double err;
7575+ stbir_uint64 top, bot;
7576+ stbir_uint64 numer_last = 0;
7577+ stbir_uint64 denom_last = 1;
7578+ stbir_uint64 numer_estimate = 1;
7579+ stbir_uint64 denom_estimate = 0;
7580+
7581+ // scale to past float error range
7582+ top = (stbir_uint64)( f * (double)(1 << 25) );
7583+ bot = 1 << 25;
7584+
7585+ // keep refining, but usually stops in a few loops - usually 5 for bad cases
7586+ for(;;)
7587+ {
7588+ stbir_uint64 est, temp;
7589+
7590+ // hit limit, break out and do best full range estimate
7591+ if ( ( ( limit_denom ) ? denom_estimate : numer_estimate ) >= limit )
7592+ break;
7593+
7594+ // is the current error less than 1 bit of a float? if so, we're done
7595+ if ( denom_estimate )
7596+ {
7597+ err = ( (double)numer_estimate / (double)denom_estimate ) - f;
7598+ if ( err < 0.0 ) err = -err;
7599+ if ( err < ( 1.0 / (double)(1<<24) ) )
7600+ {
7601+ // yup, found it
7602+ *numer = (stbir_uint32) numer_estimate;
7603+ *denom = (stbir_uint32) denom_estimate;
7604+ return 1;
7605+ }
7606+ }
7607+
7608+ // no more refinement bits left? break out and do full range estimate
7609+ if ( bot == 0 )
7610+ break;
7611+
7612+ // gcd the estimate bits
7613+ est = top / bot;
7614+ temp = top % bot;
7615+ top = bot;
7616+ bot = temp;
7617+
7618+ // move remainders
7619+ temp = est * denom_estimate + denom_last;
7620+ denom_last = denom_estimate;
7621+ denom_estimate = temp;
7622+
7623+ // move remainders
7624+ temp = est * numer_estimate + numer_last;
7625+ numer_last = numer_estimate;
7626+ numer_estimate = temp;
7627+ }
7628+
7629+ // we didn't find anything good enough for float, use a full range estimate
7630+ if ( limit_denom )
7631+ {
7632+ numer_estimate= (stbir_uint64)( f * (double)limit + 0.5 );
7633+ denom_estimate = limit;
7634+ }
7635+ else
7636+ {
7637+ numer_estimate = limit;
7638+ denom_estimate = (stbir_uint64)( ( (double)limit / f ) + 0.5 );
7639+ }
7640+
7641+ *numer = (stbir_uint32) numer_estimate;
7642+ *denom = (stbir_uint32) denom_estimate;
7643+
7644+ err = ( denom_estimate ) ? ( ( (double)(stbir_uint32)numer_estimate / (double)(stbir_uint32)denom_estimate ) - f ) : 1.0;
7645+ if ( err < 0.0 ) err = -err;
7646+ return ( err < ( 1.0 / (double)(1<<24) ) ) ? 1 : 0;
7647+}
7648+
7649+static int stbir__calculate_region_transform( stbir__scale_info * scale_info, int output_full_range, int * output_offset, int output_sub_range, int input_full_range, double input_s0, double input_s1 )
7650+{
7651+ double output_range, input_range, output_s, input_s, ratio, scale;
7652+
7653+ input_s = input_s1 - input_s0;
7654+
7655+ // null area
7656+ if ( ( output_full_range == 0 ) || ( input_full_range == 0 ) ||
7657+ ( output_sub_range == 0 ) || ( input_s <= stbir__small_float ) )
7658+ return 0;
7659+
7660+ // are either of the ranges completely out of bounds?
7661+ if ( ( *output_offset >= output_full_range ) || ( ( *output_offset + output_sub_range ) <= 0 ) || ( input_s0 >= (1.0f-stbir__small_float) ) || ( input_s1 <= stbir__small_float ) )
7662+ return 0;
7663+
7664+ output_range = (double)output_full_range;
7665+ input_range = (double)input_full_range;
7666+
7667+ output_s = ( (double)output_sub_range) / output_range;
7668+
7669+ // figure out the scaling to use
7670+ ratio = output_s / input_s;
7671+
7672+ // save scale before clipping
7673+ scale = ( output_range / input_range ) * ratio;
7674+ scale_info->scale = (float)scale;
7675+ scale_info->inv_scale = (float)( 1.0 / scale );
7676+
7677+ // clip output area to left/right output edges (and adjust input area)
7678+ stbir__clip( output_offset, &output_sub_range, output_full_range, &input_s0, &input_s1 );
7679+
7680+ // recalc input area
7681+ input_s = input_s1 - input_s0;
7682+
7683+ // after clipping do we have zero input area?
7684+ if ( input_s <= stbir__small_float )
7685+ return 0;
7686+
7687+ // calculate and store the starting source offsets in output pixel space
7688+ scale_info->pixel_shift = (float) ( input_s0 * ratio * output_range );
7689+
7690+ scale_info->scale_is_rational = stbir__double_to_rational( scale, ( scale <= 1.0 ) ? output_full_range : input_full_range, &scale_info->scale_numerator, &scale_info->scale_denominator, ( scale >= 1.0 ) );
7691+
7692+ scale_info->input_full_size = input_full_range;
7693+ scale_info->output_sub_size = output_sub_range;
7694+
7695+ return 1;
7696+}
7697+
7698+
7699+static void stbir__init_and_set_layout( STBIR_RESIZE * resize, stbir_pixel_layout pixel_layout, stbir_datatype data_type )
7700+{
7701+ resize->input_cb = 0;
7702+ resize->output_cb = 0;
7703+ resize->user_data = resize;
7704+ resize->samplers = 0;
7705+ resize->called_alloc = 0;
7706+ resize->horizontal_filter = STBIR_FILTER_DEFAULT;
7707+ resize->horizontal_filter_kernel = 0; resize->horizontal_filter_support = 0;
7708+ resize->vertical_filter = STBIR_FILTER_DEFAULT;
7709+ resize->vertical_filter_kernel = 0; resize->vertical_filter_support = 0;
7710+ resize->horizontal_edge = STBIR_EDGE_CLAMP;
7711+ resize->vertical_edge = STBIR_EDGE_CLAMP;
7712+ resize->input_s0 = 0; resize->input_t0 = 0; resize->input_s1 = 1; resize->input_t1 = 1;
7713+ resize->output_subx = 0; resize->output_suby = 0; resize->output_subw = resize->output_w; resize->output_subh = resize->output_h;
7714+ resize->input_data_type = data_type;
7715+ resize->output_data_type = data_type;
7716+ resize->input_pixel_layout_public = pixel_layout;
7717+ resize->output_pixel_layout_public = pixel_layout;
7718+ resize->needs_rebuild = 1;
7719+}
7720+
7721+STBIRDEF void stbir_resize_init( STBIR_RESIZE * resize,
7722+ const void *input_pixels, int input_w, int input_h, int input_stride_in_bytes, // stride can be zero
7723+ void *output_pixels, int output_w, int output_h, int output_stride_in_bytes, // stride can be zero
7724+ stbir_pixel_layout pixel_layout, stbir_datatype data_type )
7725+{
7726+ resize->input_pixels = input_pixels;
7727+ resize->input_w = input_w;
7728+ resize->input_h = input_h;
7729+ resize->input_stride_in_bytes = input_stride_in_bytes;
7730+ resize->output_pixels = output_pixels;
7731+ resize->output_w = output_w;
7732+ resize->output_h = output_h;
7733+ resize->output_stride_in_bytes = output_stride_in_bytes;
7734+ resize->fast_alpha = 0;
7735+
7736+ stbir__init_and_set_layout( resize, pixel_layout, data_type );
7737+}
7738+
7739+// You can update parameters any time after resize_init
7740+STBIRDEF void stbir_set_datatypes( STBIR_RESIZE * resize, stbir_datatype input_type, stbir_datatype output_type ) // by default, datatype from resize_init
7741+{
7742+ resize->input_data_type = input_type;
7743+ resize->output_data_type = output_type;
7744+ if ( ( resize->samplers ) && ( !resize->needs_rebuild ) )
7745+ stbir__update_info_from_resize( resize->samplers, resize );
7746+}
7747+
7748+STBIRDEF void stbir_set_pixel_callbacks( STBIR_RESIZE * resize, stbir_input_callback * input_cb, stbir_output_callback * output_cb ) // no callbacks by default
7749+{
7750+ resize->input_cb = input_cb;
7751+ resize->output_cb = output_cb;
7752+
7753+ if ( ( resize->samplers ) && ( !resize->needs_rebuild ) )
7754+ {
7755+ resize->samplers->in_pixels_cb = input_cb;
7756+ resize->samplers->out_pixels_cb = output_cb;
7757+ }
7758+}
7759+
7760+STBIRDEF void stbir_set_user_data( STBIR_RESIZE * resize, void * user_data ) // pass back STBIR_RESIZE* by default
7761+{
7762+ resize->user_data = user_data;
7763+ if ( ( resize->samplers ) && ( !resize->needs_rebuild ) )
7764+ resize->samplers->user_data = user_data;
7765+}
7766+
7767+STBIRDEF void stbir_set_buffer_ptrs( STBIR_RESIZE * resize, const void * input_pixels, int input_stride_in_bytes, void * output_pixels, int output_stride_in_bytes )
7768+{
7769+ resize->input_pixels = input_pixels;
7770+ resize->input_stride_in_bytes = input_stride_in_bytes;
7771+ resize->output_pixels = output_pixels;
7772+ resize->output_stride_in_bytes = output_stride_in_bytes;
7773+ if ( ( resize->samplers ) && ( !resize->needs_rebuild ) )
7774+ stbir__update_info_from_resize( resize->samplers, resize );
7775+}
7776+
7777+
7778+STBIRDEF int stbir_set_edgemodes( STBIR_RESIZE * resize, stbir_edge horizontal_edge, stbir_edge vertical_edge ) // CLAMP by default
7779+{
7780+ resize->horizontal_edge = horizontal_edge;
7781+ resize->vertical_edge = vertical_edge;
7782+ resize->needs_rebuild = 1;
7783+ return 1;
7784+}
7785+
7786+STBIRDEF int stbir_set_filters( STBIR_RESIZE * resize, stbir_filter horizontal_filter, stbir_filter vertical_filter ) // STBIR_DEFAULT_FILTER_UPSAMPLE/DOWNSAMPLE by default
7787+{
7788+ resize->horizontal_filter = horizontal_filter;
7789+ resize->vertical_filter = vertical_filter;
7790+ resize->needs_rebuild = 1;
7791+ return 1;
7792+}
7793+
7794+STBIRDEF int stbir_set_filter_callbacks( STBIR_RESIZE * resize, stbir__kernel_callback * horizontal_filter, stbir__support_callback * horizontal_support, stbir__kernel_callback * vertical_filter, stbir__support_callback * vertical_support )
7795+{
7796+ resize->horizontal_filter_kernel = horizontal_filter; resize->horizontal_filter_support = horizontal_support;
7797+ resize->vertical_filter_kernel = vertical_filter; resize->vertical_filter_support = vertical_support;
7798+ resize->needs_rebuild = 1;
7799+ return 1;
7800+}
7801+
7802+STBIRDEF int stbir_set_pixel_layouts( STBIR_RESIZE * resize, stbir_pixel_layout input_pixel_layout, stbir_pixel_layout output_pixel_layout ) // sets new pixel layouts
7803+{
7804+ resize->input_pixel_layout_public = input_pixel_layout;
7805+ resize->output_pixel_layout_public = output_pixel_layout;
7806+ resize->needs_rebuild = 1;
7807+ return 1;
7808+}
7809+
7810+
7811+STBIRDEF int stbir_set_non_pm_alpha_speed_over_quality( STBIR_RESIZE * resize, int non_pma_alpha_speed_over_quality ) // sets alpha speed
7812+{
7813+ resize->fast_alpha = non_pma_alpha_speed_over_quality;
7814+ resize->needs_rebuild = 1;
7815+ return 1;
7816+}
7817+
7818+STBIRDEF int stbir_set_input_subrect( STBIR_RESIZE * resize, double s0, double t0, double s1, double t1 ) // sets input region (full region by default)
7819+{
7820+ resize->input_s0 = s0;
7821+ resize->input_t0 = t0;
7822+ resize->input_s1 = s1;
7823+ resize->input_t1 = t1;
7824+ resize->needs_rebuild = 1;
7825+
7826+ // are we inbounds?
7827+ if ( ( s1 < stbir__small_float ) || ( (s1-s0) < stbir__small_float ) ||
7828+ ( t1 < stbir__small_float ) || ( (t1-t0) < stbir__small_float ) ||
7829+ ( s0 > (1.0f-stbir__small_float) ) ||
7830+ ( t0 > (1.0f-stbir__small_float) ) )
7831+ return 0;
7832+
7833+ return 1;
7834+}
7835+
7836+STBIRDEF int stbir_set_output_pixel_subrect( STBIR_RESIZE * resize, int subx, int suby, int subw, int subh ) // sets input region (full region by default)
7837+{
7838+ resize->output_subx = subx;
7839+ resize->output_suby = suby;
7840+ resize->output_subw = subw;
7841+ resize->output_subh = subh;
7842+ resize->needs_rebuild = 1;
7843+
7844+ // are we inbounds?
7845+ if ( ( subx >= resize->output_w ) || ( ( subx + subw ) <= 0 ) || ( suby >= resize->output_h ) || ( ( suby + subh ) <= 0 ) || ( subw == 0 ) || ( subh == 0 ) )
7846+ return 0;
7847+
7848+ return 1;
7849+}
7850+
7851+STBIRDEF int stbir_set_pixel_subrect( STBIR_RESIZE * resize, int subx, int suby, int subw, int subh ) // sets both regions (full regions by default)
7852+{
7853+ double s0, t0, s1, t1;
7854+
7855+ s0 = ( (double)subx ) / ( (double)resize->output_w );
7856+ t0 = ( (double)suby ) / ( (double)resize->output_h );
7857+ s1 = ( (double)(subx+subw) ) / ( (double)resize->output_w );
7858+ t1 = ( (double)(suby+subh) ) / ( (double)resize->output_h );
7859+
7860+ resize->input_s0 = s0;
7861+ resize->input_t0 = t0;
7862+ resize->input_s1 = s1;
7863+ resize->input_t1 = t1;
7864+ resize->output_subx = subx;
7865+ resize->output_suby = suby;
7866+ resize->output_subw = subw;
7867+ resize->output_subh = subh;
7868+ resize->needs_rebuild = 1;
7869+
7870+ // are we inbounds?
7871+ if ( ( subx >= resize->output_w ) || ( ( subx + subw ) <= 0 ) || ( suby >= resize->output_h ) || ( ( suby + subh ) <= 0 ) || ( subw == 0 ) || ( subh == 0 ) )
7872+ return 0;
7873+
7874+ return 1;
7875+}
7876+
7877+static int stbir__perform_build( STBIR_RESIZE * resize, int splits )
7878+{
7879+ stbir__contributors conservative = { 0, 0 };
7880+ stbir__sampler horizontal, vertical;
7881+ int new_output_subx, new_output_suby;
7882+ stbir__info * out_info;
7883+ #ifdef STBIR_PROFILE
7884+ stbir__info profile_infod; // used to contain building profile info before everything is allocated
7885+ stbir__info * profile_info = &profile_infod;
7886+ #endif
7887+
7888+ // have we already built the samplers?
7889+ if ( resize->samplers )
7890+ return 0;
7891+
7892+ #define STBIR_RETURN_ERROR_AND_ASSERT( exp ) STBIR_ASSERT( !(exp) ); if (exp) return 0;
7893+ STBIR_RETURN_ERROR_AND_ASSERT( (unsigned)resize->horizontal_filter >= STBIR_FILTER_OTHER)
7894+ STBIR_RETURN_ERROR_AND_ASSERT( (unsigned)resize->vertical_filter >= STBIR_FILTER_OTHER)
7895+ #undef STBIR_RETURN_ERROR_AND_ASSERT
7896+
7897+ if ( splits <= 0 )
7898+ return 0;
7899+
7900+ STBIR_PROFILE_BUILD_FIRST_START( build );
7901+
7902+ new_output_subx = resize->output_subx;
7903+ new_output_suby = resize->output_suby;
7904+
7905+ // do horizontal clip and scale calcs
7906+ if ( !stbir__calculate_region_transform( &horizontal.scale_info, resize->output_w, &new_output_subx, resize->output_subw, resize->input_w, resize->input_s0, resize->input_s1 ) )
7907+ return 0;
7908+
7909+ // do vertical clip and scale calcs
7910+ if ( !stbir__calculate_region_transform( &vertical.scale_info, resize->output_h, &new_output_suby, resize->output_subh, resize->input_h, resize->input_t0, resize->input_t1 ) )
7911+ return 0;
7912+
7913+ // if nothing to do, just return
7914+ if ( ( horizontal.scale_info.output_sub_size == 0 ) || ( vertical.scale_info.output_sub_size == 0 ) )
7915+ return 0;
7916+
7917+ stbir__set_sampler(&horizontal, resize->horizontal_filter, resize->horizontal_filter_kernel, resize->horizontal_filter_support, resize->horizontal_edge, &horizontal.scale_info, 1, resize->user_data );
7918+ stbir__get_conservative_extents( &horizontal, &conservative, resize->user_data );
7919+ stbir__set_sampler(&vertical, resize->vertical_filter, resize->vertical_filter_kernel, resize->vertical_filter_support, resize->vertical_edge, &vertical.scale_info, 0, resize->user_data );
7920+
7921+ if ( ( vertical.scale_info.output_sub_size / splits ) < STBIR_FORCE_MINIMUM_SCANLINES_FOR_SPLITS ) // each split should be a minimum of 4 scanlines (handwavey choice)
7922+ {
7923+ splits = vertical.scale_info.output_sub_size / STBIR_FORCE_MINIMUM_SCANLINES_FOR_SPLITS;
7924+ if ( splits == 0 ) splits = 1;
7925+ }
7926+
7927+ STBIR_PROFILE_BUILD_START( alloc );
7928+ out_info = stbir__alloc_internal_mem_and_build_samplers( &horizontal, &vertical, &conservative, resize->input_pixel_layout_public, resize->output_pixel_layout_public, splits, new_output_subx, new_output_suby, resize->fast_alpha, resize->user_data STBIR_ONLY_PROFILE_BUILD_SET_INFO );
7929+ STBIR_PROFILE_BUILD_END( alloc );
7930+ STBIR_PROFILE_BUILD_END( build );
7931+
7932+ if ( out_info )
7933+ {
7934+ resize->splits = splits;
7935+ resize->samplers = out_info;
7936+ resize->needs_rebuild = 0;
7937+ #ifdef STBIR_PROFILE
7938+ STBIR_MEMCPY( &out_info->profile, &profile_infod.profile, sizeof( out_info->profile ) );
7939+ #endif
7940+
7941+ // update anything that can be changed without recalcing samplers
7942+ stbir__update_info_from_resize( out_info, resize );
7943+
7944+ return splits;
7945+ }
7946+
7947+ return 0;
7948+}
7949+
7950+STBIRDEF void stbir_free_samplers( STBIR_RESIZE * resize )
7951+{
7952+ if ( resize->samplers )
7953+ {
7954+ stbir__free_internal_mem( resize->samplers );
7955+ resize->samplers = 0;
7956+ resize->called_alloc = 0;
7957+ }
7958+}
7959+
7960+STBIRDEF int stbir_build_samplers_with_splits( STBIR_RESIZE * resize, int splits )
7961+{
7962+ if ( ( resize->samplers == 0 ) || ( resize->needs_rebuild ) )
7963+ {
7964+ if ( resize->samplers )
7965+ stbir_free_samplers( resize );
7966+
7967+ resize->called_alloc = 1;
7968+ return stbir__perform_build( resize, splits );
7969+ }
7970+
7971+ STBIR_PROFILE_BUILD_CLEAR( resize->samplers );
7972+
7973+ return 1;
7974+}
7975+
7976+STBIRDEF int stbir_build_samplers( STBIR_RESIZE * resize )
7977+{
7978+ return stbir_build_samplers_with_splits( resize, 1 );
7979+}
7980+
7981+STBIRDEF int stbir_resize_extended( STBIR_RESIZE * resize )
7982+{
7983+ int result;
7984+
7985+ if ( ( resize->samplers == 0 ) || ( resize->needs_rebuild ) )
7986+ {
7987+ int alloc_state = resize->called_alloc; // remember allocated state
7988+
7989+ if ( resize->samplers )
7990+ {
7991+ stbir__free_internal_mem( resize->samplers );
7992+ resize->samplers = 0;
7993+ }
7994+
7995+ if ( !stbir_build_samplers( resize ) )
7996+ return 0;
7997+
7998+ resize->called_alloc = alloc_state;
7999+
8000+ // if build_samplers succeeded (above), but there are no samplers set, then
8001+ // the area to stretch into was zero pixels, so don't do anything and return
8002+ // success
8003+ if ( resize->samplers == 0 )
8004+ return 1;
8005+ }
8006+ else
8007+ {
8008+ // didn't build anything - clear it
8009+ STBIR_PROFILE_BUILD_CLEAR( resize->samplers );
8010+ }
8011+
8012+ // do resize
8013+ result = stbir__perform_resize( resize->samplers, 0, resize->splits );
8014+
8015+ // if we alloced, then free
8016+ if ( !resize->called_alloc )
8017+ {
8018+ stbir_free_samplers( resize );
8019+ resize->samplers = 0;
8020+ }
8021+
8022+ return result;
8023+}
8024+
8025+STBIRDEF int stbir_resize_extended_split( STBIR_RESIZE * resize, int split_start, int split_count )
8026+{
8027+ STBIR_ASSERT( resize->samplers );
8028+
8029+ // if we're just doing the whole thing, call full
8030+ if ( ( split_start == -1 ) || ( ( split_start == 0 ) && ( split_count == resize->splits ) ) )
8031+ return stbir_resize_extended( resize );
8032+
8033+ // you **must** build samplers first when using split resize
8034+ if ( ( resize->samplers == 0 ) || ( resize->needs_rebuild ) )
8035+ return 0;
8036+
8037+ if ( ( split_start >= resize->splits ) || ( split_start < 0 ) || ( ( split_start + split_count ) > resize->splits ) || ( split_count <= 0 ) )
8038+ return 0;
8039+
8040+ // do resize
8041+ return stbir__perform_resize( resize->samplers, split_start, split_count );
8042+}
8043+
8044+
8045+static void * stbir_quick_resize_helper( const void *input_pixels , int input_w , int input_h, int input_stride_in_bytes,
8046+ void *output_pixels, int output_w, int output_h, int output_stride_in_bytes,
8047+ stbir_pixel_layout pixel_layout, stbir_datatype data_type, stbir_edge edge, stbir_filter filter )
8048+{
8049+ STBIR_RESIZE resize;
8050+ int scanline_output_in_bytes;
8051+ int positive_output_stride_in_bytes;
8052+ void * start_ptr;
8053+ void * free_ptr;
8054+
8055+ scanline_output_in_bytes = output_w * stbir__type_size[ data_type ] * stbir__pixel_channels[ stbir__pixel_layout_convert_public_to_internal[ pixel_layout ] ];
8056+ if ( scanline_output_in_bytes == 0 )
8057+ return 0;
8058+
8059+ // if zero stride, use scanline output
8060+ if ( output_stride_in_bytes == 0 )
8061+ output_stride_in_bytes = scanline_output_in_bytes;
8062+
8063+ // abs value for inverted images (negative pitches)
8064+ positive_output_stride_in_bytes = output_stride_in_bytes;
8065+ if ( positive_output_stride_in_bytes < 0 )
8066+ positive_output_stride_in_bytes = -positive_output_stride_in_bytes;
8067+
8068+ // is the requested stride smaller than the scanline output? if so, just fail
8069+ if ( positive_output_stride_in_bytes < scanline_output_in_bytes )
8070+ return 0;
8071+
8072+ start_ptr = output_pixels;
8073+ free_ptr = 0; // no free pointer, since they passed buffer to use
8074+
8075+ // did they pass a zero for the dest? if so, allocate the buffer
8076+ if ( output_pixels == 0 )
8077+ {
8078+ size_t size;
8079+ char * ptr;
8080+
8081+ size = (size_t)positive_output_stride_in_bytes * (size_t)output_h;
8082+ if ( size == 0 )
8083+ return 0;
8084+
8085+ ptr = (char*) STBIR_MALLOC( size, 0 );
8086+ if ( ptr == 0 )
8087+ return 0;
8088+
8089+ free_ptr = ptr;
8090+
8091+ // point at the last scanline, if they requested a flipped image
8092+ if ( output_stride_in_bytes < 0 )
8093+ start_ptr = ptr + ( (size_t)positive_output_stride_in_bytes * (size_t)( output_h - 1 ) );
8094+ else
8095+ start_ptr = ptr;
8096+ }
8097+
8098+ // ok, now do the resize
8099+ stbir_resize_init( &resize,
8100+ input_pixels, input_w, input_h, input_stride_in_bytes,
8101+ start_ptr, output_w, output_h, output_stride_in_bytes,
8102+ pixel_layout, data_type );
8103+
8104+ resize.horizontal_edge = edge;
8105+ resize.vertical_edge = edge;
8106+ resize.horizontal_filter = filter;
8107+ resize.vertical_filter = filter;
8108+
8109+ if ( !stbir_resize_extended( &resize ) )
8110+ {
8111+ if ( free_ptr )
8112+ STBIR_FREE( free_ptr, 0 );
8113+ return 0;
8114+ }
8115+
8116+ return (free_ptr) ? free_ptr : start_ptr;
8117+}
8118+
8119+
8120+
8121+STBIRDEF unsigned char * stbir_resize_uint8_linear( const unsigned char *input_pixels , int input_w , int input_h, int input_stride_in_bytes,
8122+ unsigned char *output_pixels, int output_w, int output_h, int output_stride_in_bytes,
8123+ stbir_pixel_layout pixel_layout )
8124+{
8125+ return (unsigned char *) stbir_quick_resize_helper( input_pixels , input_w , input_h, input_stride_in_bytes,
8126+ output_pixels, output_w, output_h, output_stride_in_bytes,
8127+ pixel_layout, STBIR_TYPE_UINT8, STBIR_EDGE_CLAMP, STBIR_FILTER_DEFAULT );
8128+}
8129+
8130+STBIRDEF unsigned char * stbir_resize_uint8_srgb( const unsigned char *input_pixels , int input_w , int input_h, int input_stride_in_bytes,
8131+ unsigned char *output_pixels, int output_w, int output_h, int output_stride_in_bytes,
8132+ stbir_pixel_layout pixel_layout )
8133+{
8134+ return (unsigned char *) stbir_quick_resize_helper( input_pixels , input_w , input_h, input_stride_in_bytes,
8135+ output_pixels, output_w, output_h, output_stride_in_bytes,
8136+ pixel_layout, STBIR_TYPE_UINT8_SRGB, STBIR_EDGE_CLAMP, STBIR_FILTER_DEFAULT );
8137+}
8138+
8139+
8140+STBIRDEF float * stbir_resize_float_linear( const float *input_pixels , int input_w , int input_h, int input_stride_in_bytes,
8141+ float *output_pixels, int output_w, int output_h, int output_stride_in_bytes,
8142+ stbir_pixel_layout pixel_layout )
8143+{
8144+ return (float *) stbir_quick_resize_helper( input_pixels , input_w , input_h, input_stride_in_bytes,
8145+ output_pixels, output_w, output_h, output_stride_in_bytes,
8146+ pixel_layout, STBIR_TYPE_FLOAT, STBIR_EDGE_CLAMP, STBIR_FILTER_DEFAULT );
8147+}
8148+
8149+
8150+STBIRDEF void * stbir_resize( const void *input_pixels , int input_w , int input_h, int input_stride_in_bytes,
8151+ void *output_pixels, int output_w, int output_h, int output_stride_in_bytes,
8152+ stbir_pixel_layout pixel_layout, stbir_datatype data_type,
8153+ stbir_edge edge, stbir_filter filter )
8154+{
8155+ return (void *) stbir_quick_resize_helper( input_pixels , input_w , input_h, input_stride_in_bytes,
8156+ output_pixels, output_w, output_h, output_stride_in_bytes,
8157+ pixel_layout, data_type, edge, filter );
8158+}
8159+
8160+#ifdef STBIR_PROFILE
8161+
8162+STBIRDEF void stbir_resize_build_profile_info( STBIR_PROFILE_INFO * info, STBIR_RESIZE const * resize )
8163+{
8164+ static char const * bdescriptions[6] = { "Building", "Allocating", "Horizontal sampler", "Vertical sampler", "Coefficient cleanup", "Coefficient pivot" } ;
8165+ stbir__info* samp = resize->samplers;
8166+ int i;
8167+
8168+ typedef int testa[ (STBIR__ARRAY_SIZE( bdescriptions ) == (STBIR__ARRAY_SIZE( samp->profile.array )-1) )?1:-1];
8169+ typedef int testb[ (sizeof( samp->profile.array ) == (sizeof(samp->profile.named)) )?1:-1];
8170+ typedef int testc[ (sizeof( info->clocks ) >= (sizeof(samp->profile.named)) )?1:-1];
8171+
8172+ for( i = 0 ; i < STBIR__ARRAY_SIZE( bdescriptions ) ; i++)
8173+ info->clocks[i] = samp->profile.array[i+1];
8174+
8175+ info->total_clocks = samp->profile.named.total;
8176+ info->descriptions = bdescriptions;
8177+ info->count = STBIR__ARRAY_SIZE( bdescriptions );
8178+}
8179+
8180+STBIRDEF void stbir_resize_split_profile_info( STBIR_PROFILE_INFO * info, STBIR_RESIZE const * resize, int split_start, int split_count )
8181+{
8182+ static char const * descriptions[7] = { "Looping", "Vertical sampling", "Horizontal sampling", "Scanline input", "Scanline output", "Alpha weighting", "Alpha unweighting" };
8183+ stbir__per_split_info * split_info;
8184+ int s, i;
8185+
8186+ typedef int testa[ (STBIR__ARRAY_SIZE( descriptions ) == (STBIR__ARRAY_SIZE( split_info->profile.array )-1) )?1:-1];
8187+ typedef int testb[ (sizeof( split_info->profile.array ) == (sizeof(split_info->profile.named)) )?1:-1];
8188+ typedef int testc[ (sizeof( info->clocks ) >= (sizeof(split_info->profile.named)) )?1:-1];
8189+
8190+ if ( split_start == -1 )
8191+ {
8192+ split_start = 0;
8193+ split_count = resize->samplers->splits;
8194+ }
8195+
8196+ if ( ( split_start >= resize->splits ) || ( split_start < 0 ) || ( ( split_start + split_count ) > resize->splits ) || ( split_count <= 0 ) )
8197+ {
8198+ info->total_clocks = 0;
8199+ info->descriptions = 0;
8200+ info->count = 0;
8201+ return;
8202+ }
8203+
8204+ split_info = resize->samplers->split_info + split_start;
8205+
8206+ // sum up the profile from all the splits
8207+ for( i = 0 ; i < STBIR__ARRAY_SIZE( descriptions ) ; i++ )
8208+ {
8209+ stbir_uint64 sum = 0;
8210+ for( s = 0 ; s < split_count ; s++ )
8211+ sum += split_info[s].profile.array[i+1];
8212+ info->clocks[i] = sum;
8213+ }
8214+
8215+ info->total_clocks = split_info->profile.named.total;
8216+ info->descriptions = descriptions;
8217+ info->count = STBIR__ARRAY_SIZE( descriptions );
8218+}
8219+
8220+STBIRDEF void stbir_resize_extended_profile_info( STBIR_PROFILE_INFO * info, STBIR_RESIZE const * resize )
8221+{
8222+ stbir_resize_split_profile_info( info, resize, -1, 0 );
8223+}
8224+
8225+#endif // STBIR_PROFILE
8226+
8227+#undef STBIR_BGR
8228+#undef STBIR_1CHANNEL
8229+#undef STBIR_2CHANNEL
8230+#undef STBIR_RGB
8231+#undef STBIR_RGBA
8232+#undef STBIR_4CHANNEL
8233+#undef STBIR_BGRA
8234+#undef STBIR_ARGB
8235+#undef STBIR_ABGR
8236+#undef STBIR_RA
8237+#undef STBIR_AR
8238+#undef STBIR_RGBA_PM
8239+#undef STBIR_BGRA_PM
8240+#undef STBIR_ARGB_PM
8241+#undef STBIR_ABGR_PM
8242+#undef STBIR_RA_PM
8243+#undef STBIR_AR_PM
8244+
8245+#endif // STB_IMAGE_RESIZE_IMPLEMENTATION
8246+
8247+#else // STB_IMAGE_RESIZE_HORIZONTALS&STB_IMAGE_RESIZE_DO_VERTICALS
8248+
8249+// we reinclude the header file to define all the horizontal functions
8250+// specializing each function for the number of coeffs is 20-40% faster *OVERALL*
8251+
8252+// by including the header file again this way, we can still debug the functions
8253+
8254+#define STBIR_strs_join2( start, mid, end ) start##mid##end
8255+#define STBIR_strs_join1( start, mid, end ) STBIR_strs_join2( start, mid, end )
8256+
8257+#define STBIR_strs_join24( start, mid1, mid2, end ) start##mid1##mid2##end
8258+#define STBIR_strs_join14( start, mid1, mid2, end ) STBIR_strs_join24( start, mid1, mid2, end )
8259+
8260+#ifdef STB_IMAGE_RESIZE_DO_CODERS
8261+
8262+#ifdef stbir__decode_suffix
8263+#define STBIR__CODER_NAME( name ) STBIR_strs_join1( name, _, stbir__decode_suffix )
8264+#else
8265+#define STBIR__CODER_NAME( name ) name
8266+#endif
8267+
8268+#ifdef stbir__decode_swizzle
8269+#define stbir__decode_simdf8_flip(reg) STBIR_strs_join1( STBIR_strs_join1( STBIR_strs_join1( STBIR_strs_join1( stbir__simdf8_0123to,stbir__decode_order0,stbir__decode_order1),stbir__decode_order2,stbir__decode_order3),stbir__decode_order0,stbir__decode_order1),stbir__decode_order2,stbir__decode_order3)(reg, reg)
8270+#define stbir__decode_simdf4_flip(reg) STBIR_strs_join1( STBIR_strs_join1( stbir__simdf_0123to,stbir__decode_order0,stbir__decode_order1),stbir__decode_order2,stbir__decode_order3)(reg, reg)
8271+#define stbir__encode_simdf8_unflip(reg) STBIR_strs_join1( STBIR_strs_join1( STBIR_strs_join1( STBIR_strs_join1( stbir__simdf8_0123to,stbir__encode_order0,stbir__encode_order1),stbir__encode_order2,stbir__encode_order3),stbir__encode_order0,stbir__encode_order1),stbir__encode_order2,stbir__encode_order3)(reg, reg)
8272+#define stbir__encode_simdf4_unflip(reg) STBIR_strs_join1( STBIR_strs_join1( stbir__simdf_0123to,stbir__encode_order0,stbir__encode_order1),stbir__encode_order2,stbir__encode_order3)(reg, reg)
8273+#else
8274+#define stbir__decode_order0 0
8275+#define stbir__decode_order1 1
8276+#define stbir__decode_order2 2
8277+#define stbir__decode_order3 3
8278+#define stbir__encode_order0 0
8279+#define stbir__encode_order1 1
8280+#define stbir__encode_order2 2
8281+#define stbir__encode_order3 3
8282+#define stbir__decode_simdf8_flip(reg)
8283+#define stbir__decode_simdf4_flip(reg)
8284+#define stbir__encode_simdf8_unflip(reg)
8285+#define stbir__encode_simdf4_unflip(reg)
8286+#endif
8287+
8288+#ifdef STBIR_SIMD8
8289+#define stbir__encode_simdfX_unflip stbir__encode_simdf8_unflip
8290+#else
8291+#define stbir__encode_simdfX_unflip stbir__encode_simdf4_unflip
8292+#endif
8293+
8294+static float * STBIR__CODER_NAME( stbir__decode_uint8_linear_scaled )( float * decodep, int width_times_channels, void const * inputp )
8295+{
8296+ float STBIR_STREAMOUT_PTR( * ) decode = decodep;
8297+ float * decode_end = (float*) decode + width_times_channels;
8298+ unsigned char const * input = (unsigned char const*)inputp;
8299+
8300+ #ifdef STBIR_SIMD
8301+ unsigned char const * end_input_m16 = input + width_times_channels - 16;
8302+ if ( width_times_channels >= 16 )
8303+ {
8304+ decode_end -= 16;
8305+ STBIR_NO_UNROLL_LOOP_START_INF_FOR
8306+ for(;;)
8307+ {
8308+ #ifdef STBIR_SIMD8
8309+ stbir__simdi i; stbir__simdi8 o0,o1;
8310+ stbir__simdf8 of0, of1;
8311+ STBIR_NO_UNROLL(decode);
8312+ stbir__simdi_load( i, input );
8313+ stbir__simdi8_expand_u8_to_u32( o0, o1, i );
8314+ stbir__simdi8_convert_i32_to_float( of0, o0 );
8315+ stbir__simdi8_convert_i32_to_float( of1, o1 );
8316+ stbir__simdf8_mult( of0, of0, STBIR_max_uint8_as_float_inverted8);
8317+ stbir__simdf8_mult( of1, of1, STBIR_max_uint8_as_float_inverted8);
8318+ stbir__decode_simdf8_flip( of0 );
8319+ stbir__decode_simdf8_flip( of1 );
8320+ stbir__simdf8_store( decode + 0, of0 );
8321+ stbir__simdf8_store( decode + 8, of1 );
8322+ #else
8323+ stbir__simdi i, o0, o1, o2, o3;
8324+ stbir__simdf of0, of1, of2, of3;
8325+ STBIR_NO_UNROLL(decode);
8326+ stbir__simdi_load( i, input );
8327+ stbir__simdi_expand_u8_to_u32( o0,o1,o2,o3,i);
8328+ stbir__simdi_convert_i32_to_float( of0, o0 );
8329+ stbir__simdi_convert_i32_to_float( of1, o1 );
8330+ stbir__simdi_convert_i32_to_float( of2, o2 );
8331+ stbir__simdi_convert_i32_to_float( of3, o3 );
8332+ stbir__simdf_mult( of0, of0, STBIR__CONSTF(STBIR_max_uint8_as_float_inverted) );
8333+ stbir__simdf_mult( of1, of1, STBIR__CONSTF(STBIR_max_uint8_as_float_inverted) );
8334+ stbir__simdf_mult( of2, of2, STBIR__CONSTF(STBIR_max_uint8_as_float_inverted) );
8335+ stbir__simdf_mult( of3, of3, STBIR__CONSTF(STBIR_max_uint8_as_float_inverted) );
8336+ stbir__decode_simdf4_flip( of0 );
8337+ stbir__decode_simdf4_flip( of1 );
8338+ stbir__decode_simdf4_flip( of2 );
8339+ stbir__decode_simdf4_flip( of3 );
8340+ stbir__simdf_store( decode + 0, of0 );
8341+ stbir__simdf_store( decode + 4, of1 );
8342+ stbir__simdf_store( decode + 8, of2 );
8343+ stbir__simdf_store( decode + 12, of3 );
8344+ #endif
8345+ decode += 16;
8346+ input += 16;
8347+ if ( decode <= decode_end )
8348+ continue;
8349+ if ( decode == ( decode_end + 16 ) )
8350+ break;
8351+ decode = decode_end; // backup and do last couple
8352+ input = end_input_m16;
8353+ }
8354+ return decode_end + 16;
8355+ }
8356+ #endif
8357+
8358+ // try to do blocks of 4 when you can
8359+ #if stbir__coder_min_num != 3 // doesn't divide cleanly by four
8360+ decode += 4;
8361+ STBIR_SIMD_NO_UNROLL_LOOP_START
8362+ while( decode <= decode_end )
8363+ {
8364+ STBIR_SIMD_NO_UNROLL(decode);
8365+ decode[0-4] = ((float)(input[stbir__decode_order0])) * stbir__max_uint8_as_float_inverted;
8366+ decode[1-4] = ((float)(input[stbir__decode_order1])) * stbir__max_uint8_as_float_inverted;
8367+ decode[2-4] = ((float)(input[stbir__decode_order2])) * stbir__max_uint8_as_float_inverted;
8368+ decode[3-4] = ((float)(input[stbir__decode_order3])) * stbir__max_uint8_as_float_inverted;
8369+ decode += 4;
8370+ input += 4;
8371+ }
8372+ decode -= 4;
8373+ #endif
8374+
8375+ // do the remnants
8376+ #if stbir__coder_min_num < 4
8377+ STBIR_NO_UNROLL_LOOP_START
8378+ while( decode < decode_end )
8379+ {
8380+ STBIR_NO_UNROLL(decode);
8381+ decode[0] = ((float)(input[stbir__decode_order0])) * stbir__max_uint8_as_float_inverted;
8382+ #if stbir__coder_min_num >= 2
8383+ decode[1] = ((float)(input[stbir__decode_order1])) * stbir__max_uint8_as_float_inverted;
8384+ #endif
8385+ #if stbir__coder_min_num >= 3
8386+ decode[2] = ((float)(input[stbir__decode_order2])) * stbir__max_uint8_as_float_inverted;
8387+ #endif
8388+ decode += stbir__coder_min_num;
8389+ input += stbir__coder_min_num;
8390+ }
8391+ #endif
8392+
8393+ return decode_end;
8394+}
8395+
8396+static void STBIR__CODER_NAME( stbir__encode_uint8_linear_scaled )( void * outputp, int width_times_channels, float const * encode )
8397+{
8398+ unsigned char STBIR_SIMD_STREAMOUT_PTR( * ) output = (unsigned char *) outputp;
8399+ unsigned char * end_output = ( (unsigned char *) output ) + width_times_channels;
8400+
8401+ #ifdef STBIR_SIMD
8402+ if ( width_times_channels >= stbir__simdfX_float_count*2 )
8403+ {
8404+ float const * end_encode_m8 = encode + width_times_channels - stbir__simdfX_float_count*2;
8405+ end_output -= stbir__simdfX_float_count*2;
8406+ STBIR_NO_UNROLL_LOOP_START_INF_FOR
8407+ for(;;)
8408+ {
8409+ stbir__simdfX e0, e1;
8410+ stbir__simdi i;
8411+ STBIR_SIMD_NO_UNROLL(encode);
8412+ stbir__simdfX_madd_mem( e0, STBIR_simd_point5X, STBIR_max_uint8_as_floatX, encode );
8413+ stbir__simdfX_madd_mem( e1, STBIR_simd_point5X, STBIR_max_uint8_as_floatX, encode+stbir__simdfX_float_count );
8414+ stbir__encode_simdfX_unflip( e0 );
8415+ stbir__encode_simdfX_unflip( e1 );
8416+ #ifdef STBIR_SIMD8
8417+ stbir__simdf8_pack_to_16bytes( i, e0, e1 );
8418+ stbir__simdi_store( output, i );
8419+ #else
8420+ stbir__simdf_pack_to_8bytes( i, e0, e1 );
8421+ stbir__simdi_store2( output, i );
8422+ #endif
8423+ encode += stbir__simdfX_float_count*2;
8424+ output += stbir__simdfX_float_count*2;
8425+ if ( output <= end_output )
8426+ continue;
8427+ if ( output == ( end_output + stbir__simdfX_float_count*2 ) )
8428+ break;
8429+ output = end_output; // backup and do last couple
8430+ encode = end_encode_m8;
8431+ }
8432+ return;
8433+ }
8434+
8435+ // try to do blocks of 4 when you can
8436+ #if stbir__coder_min_num != 3 // doesn't divide cleanly by four
8437+ output += 4;
8438+ STBIR_NO_UNROLL_LOOP_START
8439+ while( output <= end_output )
8440+ {
8441+ stbir__simdf e0;
8442+ stbir__simdi i0;
8443+ STBIR_NO_UNROLL(encode);
8444+ stbir__simdf_load( e0, encode );
8445+ stbir__simdf_madd( e0, STBIR__CONSTF(STBIR_simd_point5), STBIR__CONSTF(STBIR_max_uint8_as_float), e0 );
8446+ stbir__encode_simdf4_unflip( e0 );
8447+ stbir__simdf_pack_to_8bytes( i0, e0, e0 ); // only use first 4
8448+ *(int*)(output-4) = stbir__simdi_to_int( i0 );
8449+ output += 4;
8450+ encode += 4;
8451+ }
8452+ output -= 4;
8453+ #endif
8454+
8455+ // do the remnants
8456+ #if stbir__coder_min_num < 4
8457+ STBIR_NO_UNROLL_LOOP_START
8458+ while( output < end_output )
8459+ {
8460+ stbir__simdf e0;
8461+ STBIR_NO_UNROLL(encode);
8462+ stbir__simdf_madd1_mem( e0, STBIR__CONSTF(STBIR_simd_point5), STBIR__CONSTF(STBIR_max_uint8_as_float), encode+stbir__encode_order0 ); output[0] = stbir__simdf_convert_float_to_uint8( e0 );
8463+ #if stbir__coder_min_num >= 2
8464+ stbir__simdf_madd1_mem( e0, STBIR__CONSTF(STBIR_simd_point5), STBIR__CONSTF(STBIR_max_uint8_as_float), encode+stbir__encode_order1 ); output[1] = stbir__simdf_convert_float_to_uint8( e0 );
8465+ #endif
8466+ #if stbir__coder_min_num >= 3
8467+ stbir__simdf_madd1_mem( e0, STBIR__CONSTF(STBIR_simd_point5), STBIR__CONSTF(STBIR_max_uint8_as_float), encode+stbir__encode_order2 ); output[2] = stbir__simdf_convert_float_to_uint8( e0 );
8468+ #endif
8469+ output += stbir__coder_min_num;
8470+ encode += stbir__coder_min_num;
8471+ }
8472+ #endif
8473+
8474+ #else
8475+
8476+ // try to do blocks of 4 when you can
8477+ #if stbir__coder_min_num != 3 // doesn't divide cleanly by four
8478+ output += 4;
8479+ while( output <= end_output )
8480+ {
8481+ float f;
8482+ f = encode[stbir__encode_order0] * stbir__max_uint8_as_float + 0.5f; STBIR_CLAMP(f, 0, 255); output[0-4] = (unsigned char)f;
8483+ f = encode[stbir__encode_order1] * stbir__max_uint8_as_float + 0.5f; STBIR_CLAMP(f, 0, 255); output[1-4] = (unsigned char)f;
8484+ f = encode[stbir__encode_order2] * stbir__max_uint8_as_float + 0.5f; STBIR_CLAMP(f, 0, 255); output[2-4] = (unsigned char)f;
8485+ f = encode[stbir__encode_order3] * stbir__max_uint8_as_float + 0.5f; STBIR_CLAMP(f, 0, 255); output[3-4] = (unsigned char)f;
8486+ output += 4;
8487+ encode += 4;
8488+ }
8489+ output -= 4;
8490+ #endif
8491+
8492+ // do the remnants
8493+ #if stbir__coder_min_num < 4
8494+ STBIR_NO_UNROLL_LOOP_START
8495+ while( output < end_output )
8496+ {
8497+ float f;
8498+ STBIR_NO_UNROLL(encode);
8499+ f = encode[stbir__encode_order0] * stbir__max_uint8_as_float + 0.5f; STBIR_CLAMP(f, 0, 255); output[0] = (unsigned char)f;
8500+ #if stbir__coder_min_num >= 2
8501+ f = encode[stbir__encode_order1] * stbir__max_uint8_as_float + 0.5f; STBIR_CLAMP(f, 0, 255); output[1] = (unsigned char)f;
8502+ #endif
8503+ #if stbir__coder_min_num >= 3
8504+ f = encode[stbir__encode_order2] * stbir__max_uint8_as_float + 0.5f; STBIR_CLAMP(f, 0, 255); output[2] = (unsigned char)f;
8505+ #endif
8506+ output += stbir__coder_min_num;
8507+ encode += stbir__coder_min_num;
8508+ }
8509+ #endif
8510+ #endif
8511+}
8512+
8513+static float * STBIR__CODER_NAME(stbir__decode_uint8_linear)( float * decodep, int width_times_channels, void const * inputp )
8514+{
8515+ float STBIR_STREAMOUT_PTR( * ) decode = decodep;
8516+ float * decode_end = (float*) decode + width_times_channels;
8517+ unsigned char const * input = (unsigned char const*)inputp;
8518+
8519+ #ifdef STBIR_SIMD
8520+ unsigned char const * end_input_m16 = input + width_times_channels - 16;
8521+ if ( width_times_channels >= 16 )
8522+ {
8523+ decode_end -= 16;
8524+ STBIR_NO_UNROLL_LOOP_START_INF_FOR
8525+ for(;;)
8526+ {
8527+ #ifdef STBIR_SIMD8
8528+ stbir__simdi i; stbir__simdi8 o0,o1;
8529+ stbir__simdf8 of0, of1;
8530+ STBIR_NO_UNROLL(decode);
8531+ stbir__simdi_load( i, input );
8532+ stbir__simdi8_expand_u8_to_u32( o0, o1, i );
8533+ stbir__simdi8_convert_i32_to_float( of0, o0 );
8534+ stbir__simdi8_convert_i32_to_float( of1, o1 );
8535+ stbir__decode_simdf8_flip( of0 );
8536+ stbir__decode_simdf8_flip( of1 );
8537+ stbir__simdf8_store( decode + 0, of0 );
8538+ stbir__simdf8_store( decode + 8, of1 );
8539+ #else
8540+ stbir__simdi i, o0, o1, o2, o3;
8541+ stbir__simdf of0, of1, of2, of3;
8542+ STBIR_NO_UNROLL(decode);
8543+ stbir__simdi_load( i, input );
8544+ stbir__simdi_expand_u8_to_u32( o0,o1,o2,o3,i);
8545+ stbir__simdi_convert_i32_to_float( of0, o0 );
8546+ stbir__simdi_convert_i32_to_float( of1, o1 );
8547+ stbir__simdi_convert_i32_to_float( of2, o2 );
8548+ stbir__simdi_convert_i32_to_float( of3, o3 );
8549+ stbir__decode_simdf4_flip( of0 );
8550+ stbir__decode_simdf4_flip( of1 );
8551+ stbir__decode_simdf4_flip( of2 );
8552+ stbir__decode_simdf4_flip( of3 );
8553+ stbir__simdf_store( decode + 0, of0 );
8554+ stbir__simdf_store( decode + 4, of1 );
8555+ stbir__simdf_store( decode + 8, of2 );
8556+ stbir__simdf_store( decode + 12, of3 );
8557+#endif
8558+ decode += 16;
8559+ input += 16;
8560+ if ( decode <= decode_end )
8561+ continue;
8562+ if ( decode == ( decode_end + 16 ) )
8563+ break;
8564+ decode = decode_end; // backup and do last couple
8565+ input = end_input_m16;
8566+ }
8567+ return decode_end + 16;
8568+ }
8569+ #endif
8570+
8571+ // try to do blocks of 4 when you can
8572+ #if stbir__coder_min_num != 3 // doesn't divide cleanly by four
8573+ decode += 4;
8574+ STBIR_SIMD_NO_UNROLL_LOOP_START
8575+ while( decode <= decode_end )
8576+ {
8577+ STBIR_SIMD_NO_UNROLL(decode);
8578+ decode[0-4] = ((float)(input[stbir__decode_order0]));
8579+ decode[1-4] = ((float)(input[stbir__decode_order1]));
8580+ decode[2-4] = ((float)(input[stbir__decode_order2]));
8581+ decode[3-4] = ((float)(input[stbir__decode_order3]));
8582+ decode += 4;
8583+ input += 4;
8584+ }
8585+ decode -= 4;
8586+ #endif
8587+
8588+ // do the remnants
8589+ #if stbir__coder_min_num < 4
8590+ STBIR_NO_UNROLL_LOOP_START
8591+ while( decode < decode_end )
8592+ {
8593+ STBIR_NO_UNROLL(decode);
8594+ decode[0] = ((float)(input[stbir__decode_order0]));
8595+ #if stbir__coder_min_num >= 2
8596+ decode[1] = ((float)(input[stbir__decode_order1]));
8597+ #endif
8598+ #if stbir__coder_min_num >= 3
8599+ decode[2] = ((float)(input[stbir__decode_order2]));
8600+ #endif
8601+ decode += stbir__coder_min_num;
8602+ input += stbir__coder_min_num;
8603+ }
8604+ #endif
8605+ return decode_end;
8606+}
8607+
8608+static void STBIR__CODER_NAME( stbir__encode_uint8_linear )( void * outputp, int width_times_channels, float const * encode )
8609+{
8610+ unsigned char STBIR_SIMD_STREAMOUT_PTR( * ) output = (unsigned char *) outputp;
8611+ unsigned char * end_output = ( (unsigned char *) output ) + width_times_channels;
8612+
8613+ #ifdef STBIR_SIMD
8614+ if ( width_times_channels >= stbir__simdfX_float_count*2 )
8615+ {
8616+ float const * end_encode_m8 = encode + width_times_channels - stbir__simdfX_float_count*2;
8617+ end_output -= stbir__simdfX_float_count*2;
8618+ STBIR_SIMD_NO_UNROLL_LOOP_START_INF_FOR
8619+ for(;;)
8620+ {
8621+ stbir__simdfX e0, e1;
8622+ stbir__simdi i;
8623+ STBIR_SIMD_NO_UNROLL(encode);
8624+ stbir__simdfX_add_mem( e0, STBIR_simd_point5X, encode );
8625+ stbir__simdfX_add_mem( e1, STBIR_simd_point5X, encode+stbir__simdfX_float_count );
8626+ stbir__encode_simdfX_unflip( e0 );
8627+ stbir__encode_simdfX_unflip( e1 );
8628+ #ifdef STBIR_SIMD8
8629+ stbir__simdf8_pack_to_16bytes( i, e0, e1 );
8630+ stbir__simdi_store( output, i );
8631+ #else
8632+ stbir__simdf_pack_to_8bytes( i, e0, e1 );
8633+ stbir__simdi_store2( output, i );
8634+ #endif
8635+ encode += stbir__simdfX_float_count*2;
8636+ output += stbir__simdfX_float_count*2;
8637+ if ( output <= end_output )
8638+ continue;
8639+ if ( output == ( end_output + stbir__simdfX_float_count*2 ) )
8640+ break;
8641+ output = end_output; // backup and do last couple
8642+ encode = end_encode_m8;
8643+ }
8644+ return;
8645+ }
8646+
8647+ // try to do blocks of 4 when you can
8648+ #if stbir__coder_min_num != 3 // doesn't divide cleanly by four
8649+ output += 4;
8650+ STBIR_NO_UNROLL_LOOP_START
8651+ while( output <= end_output )
8652+ {
8653+ stbir__simdf e0;
8654+ stbir__simdi i0;
8655+ STBIR_NO_UNROLL(encode);
8656+ stbir__simdf_load( e0, encode );
8657+ stbir__simdf_add( e0, STBIR__CONSTF(STBIR_simd_point5), e0 );
8658+ stbir__encode_simdf4_unflip( e0 );
8659+ stbir__simdf_pack_to_8bytes( i0, e0, e0 ); // only use first 4
8660+ *(int*)(output-4) = stbir__simdi_to_int( i0 );
8661+ output += 4;
8662+ encode += 4;
8663+ }
8664+ output -= 4;
8665+ #endif
8666+
8667+ #else
8668+
8669+ // try to do blocks of 4 when you can
8670+ #if stbir__coder_min_num != 3 // doesn't divide cleanly by four
8671+ output += 4;
8672+ while( output <= end_output )
8673+ {
8674+ float f;
8675+ f = encode[stbir__encode_order0] + 0.5f; STBIR_CLAMP(f, 0, 255); output[0-4] = (unsigned char)f;
8676+ f = encode[stbir__encode_order1] + 0.5f; STBIR_CLAMP(f, 0, 255); output[1-4] = (unsigned char)f;
8677+ f = encode[stbir__encode_order2] + 0.5f; STBIR_CLAMP(f, 0, 255); output[2-4] = (unsigned char)f;
8678+ f = encode[stbir__encode_order3] + 0.5f; STBIR_CLAMP(f, 0, 255); output[3-4] = (unsigned char)f;
8679+ output += 4;
8680+ encode += 4;
8681+ }
8682+ output -= 4;
8683+ #endif
8684+
8685+ #endif
8686+
8687+ // do the remnants
8688+ #if stbir__coder_min_num < 4
8689+ STBIR_NO_UNROLL_LOOP_START
8690+ while( output < end_output )
8691+ {
8692+ float f;
8693+ STBIR_NO_UNROLL(encode);
8694+ f = encode[stbir__encode_order0] + 0.5f; STBIR_CLAMP(f, 0, 255); output[0] = (unsigned char)f;
8695+ #if stbir__coder_min_num >= 2
8696+ f = encode[stbir__encode_order1] + 0.5f; STBIR_CLAMP(f, 0, 255); output[1] = (unsigned char)f;
8697+ #endif
8698+ #if stbir__coder_min_num >= 3
8699+ f = encode[stbir__encode_order2] + 0.5f; STBIR_CLAMP(f, 0, 255); output[2] = (unsigned char)f;
8700+ #endif
8701+ output += stbir__coder_min_num;
8702+ encode += stbir__coder_min_num;
8703+ }
8704+ #endif
8705+}
8706+
8707+static float * STBIR__CODER_NAME(stbir__decode_uint8_srgb)( float * decodep, int width_times_channels, void const * inputp )
8708+{
8709+ float STBIR_STREAMOUT_PTR( * ) decode = decodep;
8710+ float * decode_end = (float*) decode + width_times_channels;
8711+ unsigned char const * input = (unsigned char const *)inputp;
8712+
8713+ // try to do blocks of 4 when you can
8714+ #if stbir__coder_min_num != 3 // doesn't divide cleanly by four
8715+ decode += 4;
8716+ while( decode <= decode_end )
8717+ {
8718+ decode[0-4] = stbir__srgb_uchar_to_linear_float[ input[ stbir__decode_order0 ] ];
8719+ decode[1-4] = stbir__srgb_uchar_to_linear_float[ input[ stbir__decode_order1 ] ];
8720+ decode[2-4] = stbir__srgb_uchar_to_linear_float[ input[ stbir__decode_order2 ] ];
8721+ decode[3-4] = stbir__srgb_uchar_to_linear_float[ input[ stbir__decode_order3 ] ];
8722+ decode += 4;
8723+ input += 4;
8724+ }
8725+ decode -= 4;
8726+ #endif
8727+
8728+ // do the remnants
8729+ #if stbir__coder_min_num < 4
8730+ STBIR_NO_UNROLL_LOOP_START
8731+ while( decode < decode_end )
8732+ {
8733+ STBIR_NO_UNROLL(decode);
8734+ decode[0] = stbir__srgb_uchar_to_linear_float[ input[ stbir__decode_order0 ] ];
8735+ #if stbir__coder_min_num >= 2
8736+ decode[1] = stbir__srgb_uchar_to_linear_float[ input[ stbir__decode_order1 ] ];
8737+ #endif
8738+ #if stbir__coder_min_num >= 3
8739+ decode[2] = stbir__srgb_uchar_to_linear_float[ input[ stbir__decode_order2 ] ];
8740+ #endif
8741+ decode += stbir__coder_min_num;
8742+ input += stbir__coder_min_num;
8743+ }
8744+ #endif
8745+ return decode_end;
8746+}
8747+
8748+#define stbir__min_max_shift20( i, f ) \
8749+ stbir__simdf_max( f, f, stbir_simdf_casti(STBIR__CONSTI( STBIR_almost_zero )) ); \
8750+ stbir__simdf_min( f, f, stbir_simdf_casti(STBIR__CONSTI( STBIR_almost_one )) ); \
8751+ stbir__simdi_32shr( i, stbir_simdi_castf( f ), 20 );
8752+
8753+#define stbir__scale_and_convert( i, f ) \
8754+ stbir__simdf_madd( f, STBIR__CONSTF( STBIR_simd_point5 ), STBIR__CONSTF( STBIR_max_uint8_as_float ), f ); \
8755+ stbir__simdf_max( f, f, stbir__simdf_zeroP() ); \
8756+ stbir__simdf_min( f, f, STBIR__CONSTF( STBIR_max_uint8_as_float ) ); \
8757+ stbir__simdf_convert_float_to_i32( i, f );
8758+
8759+#define stbir__linear_to_srgb_finish( i, f ) \
8760+{ \
8761+ stbir__simdi temp; \
8762+ stbir__simdi_32shr( temp, stbir_simdi_castf( f ), 12 ) ; \
8763+ stbir__simdi_and( temp, temp, STBIR__CONSTI(STBIR_mantissa_mask) ); \
8764+ stbir__simdi_or( temp, temp, STBIR__CONSTI(STBIR_topscale) ); \
8765+ stbir__simdi_16madd( i, i, temp ); \
8766+ stbir__simdi_32shr( i, i, 16 ); \
8767+}
8768+
8769+#define stbir__simdi_table_lookup2( v0,v1, table ) \
8770+{ \
8771+ stbir__simdi_u32 temp0,temp1; \
8772+ temp0.m128i_i128 = v0; \
8773+ temp1.m128i_i128 = v1; \
8774+ temp0.m128i_u32[0] = table[temp0.m128i_i32[0]]; temp0.m128i_u32[1] = table[temp0.m128i_i32[1]]; temp0.m128i_u32[2] = table[temp0.m128i_i32[2]]; temp0.m128i_u32[3] = table[temp0.m128i_i32[3]]; \
8775+ temp1.m128i_u32[0] = table[temp1.m128i_i32[0]]; temp1.m128i_u32[1] = table[temp1.m128i_i32[1]]; temp1.m128i_u32[2] = table[temp1.m128i_i32[2]]; temp1.m128i_u32[3] = table[temp1.m128i_i32[3]]; \
8776+ v0 = temp0.m128i_i128; \
8777+ v1 = temp1.m128i_i128; \
8778+}
8779+
8780+#define stbir__simdi_table_lookup3( v0,v1,v2, table ) \
8781+{ \
8782+ stbir__simdi_u32 temp0,temp1,temp2; \
8783+ temp0.m128i_i128 = v0; \
8784+ temp1.m128i_i128 = v1; \
8785+ temp2.m128i_i128 = v2; \
8786+ temp0.m128i_u32[0] = table[temp0.m128i_i32[0]]; temp0.m128i_u32[1] = table[temp0.m128i_i32[1]]; temp0.m128i_u32[2] = table[temp0.m128i_i32[2]]; temp0.m128i_u32[3] = table[temp0.m128i_i32[3]]; \
8787+ temp1.m128i_u32[0] = table[temp1.m128i_i32[0]]; temp1.m128i_u32[1] = table[temp1.m128i_i32[1]]; temp1.m128i_u32[2] = table[temp1.m128i_i32[2]]; temp1.m128i_u32[3] = table[temp1.m128i_i32[3]]; \
8788+ temp2.m128i_u32[0] = table[temp2.m128i_i32[0]]; temp2.m128i_u32[1] = table[temp2.m128i_i32[1]]; temp2.m128i_u32[2] = table[temp2.m128i_i32[2]]; temp2.m128i_u32[3] = table[temp2.m128i_i32[3]]; \
8789+ v0 = temp0.m128i_i128; \
8790+ v1 = temp1.m128i_i128; \
8791+ v2 = temp2.m128i_i128; \
8792+}
8793+
8794+#define stbir__simdi_table_lookup4( v0,v1,v2,v3, table ) \
8795+{ \
8796+ stbir__simdi_u32 temp0,temp1,temp2,temp3; \
8797+ temp0.m128i_i128 = v0; \
8798+ temp1.m128i_i128 = v1; \
8799+ temp2.m128i_i128 = v2; \
8800+ temp3.m128i_i128 = v3; \
8801+ temp0.m128i_u32[0] = table[temp0.m128i_i32[0]]; temp0.m128i_u32[1] = table[temp0.m128i_i32[1]]; temp0.m128i_u32[2] = table[temp0.m128i_i32[2]]; temp0.m128i_u32[3] = table[temp0.m128i_i32[3]]; \
8802+ temp1.m128i_u32[0] = table[temp1.m128i_i32[0]]; temp1.m128i_u32[1] = table[temp1.m128i_i32[1]]; temp1.m128i_u32[2] = table[temp1.m128i_i32[2]]; temp1.m128i_u32[3] = table[temp1.m128i_i32[3]]; \
8803+ temp2.m128i_u32[0] = table[temp2.m128i_i32[0]]; temp2.m128i_u32[1] = table[temp2.m128i_i32[1]]; temp2.m128i_u32[2] = table[temp2.m128i_i32[2]]; temp2.m128i_u32[3] = table[temp2.m128i_i32[3]]; \
8804+ temp3.m128i_u32[0] = table[temp3.m128i_i32[0]]; temp3.m128i_u32[1] = table[temp3.m128i_i32[1]]; temp3.m128i_u32[2] = table[temp3.m128i_i32[2]]; temp3.m128i_u32[3] = table[temp3.m128i_i32[3]]; \
8805+ v0 = temp0.m128i_i128; \
8806+ v1 = temp1.m128i_i128; \
8807+ v2 = temp2.m128i_i128; \
8808+ v3 = temp3.m128i_i128; \
8809+}
8810+
8811+static void STBIR__CODER_NAME( stbir__encode_uint8_srgb )( void * outputp, int width_times_channels, float const * encode )
8812+{
8813+ unsigned char STBIR_SIMD_STREAMOUT_PTR( * ) output = (unsigned char*) outputp;
8814+ unsigned char * end_output = ( (unsigned char*) output ) + width_times_channels;
8815+
8816+ #ifdef STBIR_SIMD
8817+
8818+ if ( width_times_channels >= 16 )
8819+ {
8820+ float const * end_encode_m16 = encode + width_times_channels - 16;
8821+ end_output -= 16;
8822+ STBIR_SIMD_NO_UNROLL_LOOP_START_INF_FOR
8823+ for(;;)
8824+ {
8825+ stbir__simdf f0, f1, f2, f3;
8826+ stbir__simdi i0, i1, i2, i3;
8827+ STBIR_SIMD_NO_UNROLL(encode);
8828+
8829+ stbir__simdf_load4_transposed( f0, f1, f2, f3, encode );
8830+
8831+ stbir__min_max_shift20( i0, f0 );
8832+ stbir__min_max_shift20( i1, f1 );
8833+ stbir__min_max_shift20( i2, f2 );
8834+ stbir__min_max_shift20( i3, f3 );
8835+
8836+ stbir__simdi_table_lookup4( i0, i1, i2, i3, ( fp32_to_srgb8_tab4 - (127-13)*8 ) );
8837+
8838+ stbir__linear_to_srgb_finish( i0, f0 );
8839+ stbir__linear_to_srgb_finish( i1, f1 );
8840+ stbir__linear_to_srgb_finish( i2, f2 );
8841+ stbir__linear_to_srgb_finish( i3, f3 );
8842+
8843+ stbir__interleave_pack_and_store_16_u8( output, STBIR_strs_join1(i, ,stbir__encode_order0), STBIR_strs_join1(i, ,stbir__encode_order1), STBIR_strs_join1(i, ,stbir__encode_order2), STBIR_strs_join1(i, ,stbir__encode_order3) );
8844+
8845+ encode += 16;
8846+ output += 16;
8847+ if ( output <= end_output )
8848+ continue;
8849+ if ( output == ( end_output + 16 ) )
8850+ break;
8851+ output = end_output; // backup and do last couple
8852+ encode = end_encode_m16;
8853+ }
8854+ return;
8855+ }
8856+ #endif
8857+
8858+ // try to do blocks of 4 when you can
8859+ #if stbir__coder_min_num != 3 // doesn't divide cleanly by four
8860+ output += 4;
8861+ STBIR_SIMD_NO_UNROLL_LOOP_START
8862+ while ( output <= end_output )
8863+ {
8864+ STBIR_SIMD_NO_UNROLL(encode);
8865+
8866+ output[0-4] = stbir__linear_to_srgb_uchar( encode[stbir__encode_order0] );
8867+ output[1-4] = stbir__linear_to_srgb_uchar( encode[stbir__encode_order1] );
8868+ output[2-4] = stbir__linear_to_srgb_uchar( encode[stbir__encode_order2] );
8869+ output[3-4] = stbir__linear_to_srgb_uchar( encode[stbir__encode_order3] );
8870+
8871+ output += 4;
8872+ encode += 4;
8873+ }
8874+ output -= 4;
8875+ #endif
8876+
8877+ // do the remnants
8878+ #if stbir__coder_min_num < 4
8879+ STBIR_NO_UNROLL_LOOP_START
8880+ while( output < end_output )
8881+ {
8882+ STBIR_NO_UNROLL(encode);
8883+ output[0] = stbir__linear_to_srgb_uchar( encode[stbir__encode_order0] );
8884+ #if stbir__coder_min_num >= 2
8885+ output[1] = stbir__linear_to_srgb_uchar( encode[stbir__encode_order1] );
8886+ #endif
8887+ #if stbir__coder_min_num >= 3
8888+ output[2] = stbir__linear_to_srgb_uchar( encode[stbir__encode_order2] );
8889+ #endif
8890+ output += stbir__coder_min_num;
8891+ encode += stbir__coder_min_num;
8892+ }
8893+ #endif
8894+}
8895+
8896+#if ( stbir__coder_min_num == 4 ) || ( ( stbir__coder_min_num == 1 ) && ( !defined(stbir__decode_swizzle) ) )
8897+
8898+static float * STBIR__CODER_NAME(stbir__decode_uint8_srgb4_linearalpha)( float * decodep, int width_times_channels, void const * inputp )
8899+{
8900+ float STBIR_STREAMOUT_PTR( * ) decode = decodep;
8901+ float * decode_end = (float*) decode + width_times_channels;
8902+ unsigned char const * input = (unsigned char const *)inputp;
8903+
8904+ do {
8905+ decode[0] = stbir__srgb_uchar_to_linear_float[ input[stbir__decode_order0] ];
8906+ decode[1] = stbir__srgb_uchar_to_linear_float[ input[stbir__decode_order1] ];
8907+ decode[2] = stbir__srgb_uchar_to_linear_float[ input[stbir__decode_order2] ];
8908+ decode[3] = ( (float) input[stbir__decode_order3] ) * stbir__max_uint8_as_float_inverted;
8909+ input += 4;
8910+ decode += 4;
8911+ } while( decode < decode_end );
8912+ return decode_end;
8913+}
8914+
8915+
8916+static void STBIR__CODER_NAME( stbir__encode_uint8_srgb4_linearalpha )( void * outputp, int width_times_channels, float const * encode )
8917+{
8918+ unsigned char STBIR_SIMD_STREAMOUT_PTR( * ) output = (unsigned char*) outputp;
8919+ unsigned char * end_output = ( (unsigned char*) output ) + width_times_channels;
8920+
8921+ #ifdef STBIR_SIMD
8922+
8923+ if ( width_times_channels >= 16 )
8924+ {
8925+ float const * end_encode_m16 = encode + width_times_channels - 16;
8926+ end_output -= 16;
8927+ STBIR_SIMD_NO_UNROLL_LOOP_START_INF_FOR
8928+ for(;;)
8929+ {
8930+ stbir__simdf f0, f1, f2, f3;
8931+ stbir__simdi i0, i1, i2, i3;
8932+
8933+ STBIR_SIMD_NO_UNROLL(encode);
8934+ stbir__simdf_load4_transposed( f0, f1, f2, f3, encode );
8935+
8936+ stbir__min_max_shift20( i0, f0 );
8937+ stbir__min_max_shift20( i1, f1 );
8938+ stbir__min_max_shift20( i2, f2 );
8939+ stbir__scale_and_convert( i3, f3 );
8940+
8941+ stbir__simdi_table_lookup3( i0, i1, i2, ( fp32_to_srgb8_tab4 - (127-13)*8 ) );
8942+
8943+ stbir__linear_to_srgb_finish( i0, f0 );
8944+ stbir__linear_to_srgb_finish( i1, f1 );
8945+ stbir__linear_to_srgb_finish( i2, f2 );
8946+
8947+ stbir__interleave_pack_and_store_16_u8( output, STBIR_strs_join1(i, ,stbir__encode_order0), STBIR_strs_join1(i, ,stbir__encode_order1), STBIR_strs_join1(i, ,stbir__encode_order2), STBIR_strs_join1(i, ,stbir__encode_order3) );
8948+
8949+ output += 16;
8950+ encode += 16;
8951+
8952+ if ( output <= end_output )
8953+ continue;
8954+ if ( output == ( end_output + 16 ) )
8955+ break;
8956+ output = end_output; // backup and do last couple
8957+ encode = end_encode_m16;
8958+ }
8959+ return;
8960+ }
8961+ #endif
8962+
8963+ STBIR_SIMD_NO_UNROLL_LOOP_START
8964+ do {
8965+ float f;
8966+ STBIR_SIMD_NO_UNROLL(encode);
8967+
8968+ output[stbir__decode_order0] = stbir__linear_to_srgb_uchar( encode[0] );
8969+ output[stbir__decode_order1] = stbir__linear_to_srgb_uchar( encode[1] );
8970+ output[stbir__decode_order2] = stbir__linear_to_srgb_uchar( encode[2] );
8971+
8972+ f = encode[3] * stbir__max_uint8_as_float + 0.5f;
8973+ STBIR_CLAMP(f, 0, 255);
8974+ output[stbir__decode_order3] = (unsigned char) f;
8975+
8976+ output += 4;
8977+ encode += 4;
8978+ } while( output < end_output );
8979+}
8980+
8981+#endif
8982+
8983+#if ( stbir__coder_min_num == 2 ) || ( ( stbir__coder_min_num == 1 ) && ( !defined(stbir__decode_swizzle) ) )
8984+
8985+static float * STBIR__CODER_NAME(stbir__decode_uint8_srgb2_linearalpha)( float * decodep, int width_times_channels, void const * inputp )
8986+{
8987+ float STBIR_STREAMOUT_PTR( * ) decode = decodep;
8988+ float * decode_end = (float*) decode + width_times_channels;
8989+ unsigned char const * input = (unsigned char const *)inputp;
8990+
8991+ decode += 4;
8992+ while( decode <= decode_end )
8993+ {
8994+ decode[0-4] = stbir__srgb_uchar_to_linear_float[ input[stbir__decode_order0] ];
8995+ decode[1-4] = ( (float) input[stbir__decode_order1] ) * stbir__max_uint8_as_float_inverted;
8996+ decode[2-4] = stbir__srgb_uchar_to_linear_float[ input[stbir__decode_order0+2] ];
8997+ decode[3-4] = ( (float) input[stbir__decode_order1+2] ) * stbir__max_uint8_as_float_inverted;
8998+ input += 4;
8999+ decode += 4;
9000+ }
9001+ decode -= 4;
9002+ if( decode < decode_end )
9003+ {
9004+ decode[0] = stbir__srgb_uchar_to_linear_float[ input[stbir__decode_order0] ];
9005+ decode[1] = ( (float) input[stbir__decode_order1] ) * stbir__max_uint8_as_float_inverted;
9006+ }
9007+ return decode_end;
9008+}
9009+
9010+static void STBIR__CODER_NAME( stbir__encode_uint8_srgb2_linearalpha )( void * outputp, int width_times_channels, float const * encode )
9011+{
9012+ unsigned char STBIR_SIMD_STREAMOUT_PTR( * ) output = (unsigned char*) outputp;
9013+ unsigned char * end_output = ( (unsigned char*) output ) + width_times_channels;
9014+
9015+ #ifdef STBIR_SIMD
9016+
9017+ if ( width_times_channels >= 16 )
9018+ {
9019+ float const * end_encode_m16 = encode + width_times_channels - 16;
9020+ end_output -= 16;
9021+ STBIR_SIMD_NO_UNROLL_LOOP_START_INF_FOR
9022+ for(;;)
9023+ {
9024+ stbir__simdf f0, f1, f2, f3;
9025+ stbir__simdi i0, i1, i2, i3;
9026+
9027+ STBIR_SIMD_NO_UNROLL(encode);
9028+ stbir__simdf_load4_transposed( f0, f1, f2, f3, encode );
9029+
9030+ stbir__min_max_shift20( i0, f0 );
9031+ stbir__scale_and_convert( i1, f1 );
9032+ stbir__min_max_shift20( i2, f2 );
9033+ stbir__scale_and_convert( i3, f3 );
9034+
9035+ stbir__simdi_table_lookup2( i0, i2, ( fp32_to_srgb8_tab4 - (127-13)*8 ) );
9036+
9037+ stbir__linear_to_srgb_finish( i0, f0 );
9038+ stbir__linear_to_srgb_finish( i2, f2 );
9039+
9040+ stbir__interleave_pack_and_store_16_u8( output, STBIR_strs_join1(i, ,stbir__encode_order0), STBIR_strs_join1(i, ,stbir__encode_order1), STBIR_strs_join1(i, ,stbir__encode_order2), STBIR_strs_join1(i, ,stbir__encode_order3) );
9041+
9042+ output += 16;
9043+ encode += 16;
9044+ if ( output <= end_output )
9045+ continue;
9046+ if ( output == ( end_output + 16 ) )
9047+ break;
9048+ output = end_output; // backup and do last couple
9049+ encode = end_encode_m16;
9050+ }
9051+ return;
9052+ }
9053+ #endif
9054+
9055+ STBIR_SIMD_NO_UNROLL_LOOP_START
9056+ do {
9057+ float f;
9058+ STBIR_SIMD_NO_UNROLL(encode);
9059+
9060+ output[stbir__decode_order0] = stbir__linear_to_srgb_uchar( encode[0] );
9061+
9062+ f = encode[1] * stbir__max_uint8_as_float + 0.5f;
9063+ STBIR_CLAMP(f, 0, 255);
9064+ output[stbir__decode_order1] = (unsigned char) f;
9065+
9066+ output += 2;
9067+ encode += 2;
9068+ } while( output < end_output );
9069+}
9070+
9071+#endif
9072+
9073+static float * STBIR__CODER_NAME(stbir__decode_uint16_linear_scaled)( float * decodep, int width_times_channels, void const * inputp )
9074+{
9075+ float STBIR_STREAMOUT_PTR( * ) decode = decodep;
9076+ float * decode_end = (float*) decode + width_times_channels;
9077+ unsigned short const * input = (unsigned short const *)inputp;
9078+
9079+ #ifdef STBIR_SIMD
9080+ unsigned short const * end_input_m8 = input + width_times_channels - 8;
9081+ if ( width_times_channels >= 8 )
9082+ {
9083+ decode_end -= 8;
9084+ STBIR_NO_UNROLL_LOOP_START_INF_FOR
9085+ for(;;)
9086+ {
9087+ #ifdef STBIR_SIMD8
9088+ stbir__simdi i; stbir__simdi8 o;
9089+ stbir__simdf8 of;
9090+ STBIR_NO_UNROLL(decode);
9091+ stbir__simdi_load( i, input );
9092+ stbir__simdi8_expand_u16_to_u32( o, i );
9093+ stbir__simdi8_convert_i32_to_float( of, o );
9094+ stbir__simdf8_mult( of, of, STBIR_max_uint16_as_float_inverted8);
9095+ stbir__decode_simdf8_flip( of );
9096+ stbir__simdf8_store( decode + 0, of );
9097+ #else
9098+ stbir__simdi i, o0, o1;
9099+ stbir__simdf of0, of1;
9100+ STBIR_NO_UNROLL(decode);
9101+ stbir__simdi_load( i, input );
9102+ stbir__simdi_expand_u16_to_u32( o0,o1,i );
9103+ stbir__simdi_convert_i32_to_float( of0, o0 );
9104+ stbir__simdi_convert_i32_to_float( of1, o1 );
9105+ stbir__simdf_mult( of0, of0, STBIR__CONSTF(STBIR_max_uint16_as_float_inverted) );
9106+ stbir__simdf_mult( of1, of1, STBIR__CONSTF(STBIR_max_uint16_as_float_inverted));
9107+ stbir__decode_simdf4_flip( of0 );
9108+ stbir__decode_simdf4_flip( of1 );
9109+ stbir__simdf_store( decode + 0, of0 );
9110+ stbir__simdf_store( decode + 4, of1 );
9111+ #endif
9112+ decode += 8;
9113+ input += 8;
9114+ if ( decode <= decode_end )
9115+ continue;
9116+ if ( decode == ( decode_end + 8 ) )
9117+ break;
9118+ decode = decode_end; // backup and do last couple
9119+ input = end_input_m8;
9120+ }
9121+ return decode_end + 8;
9122+ }
9123+ #endif
9124+
9125+ // try to do blocks of 4 when you can
9126+ #if stbir__coder_min_num != 3 // doesn't divide cleanly by four
9127+ decode += 4;
9128+ STBIR_SIMD_NO_UNROLL_LOOP_START
9129+ while( decode <= decode_end )
9130+ {
9131+ STBIR_SIMD_NO_UNROLL(decode);
9132+ decode[0-4] = ((float)(input[stbir__decode_order0])) * stbir__max_uint16_as_float_inverted;
9133+ decode[1-4] = ((float)(input[stbir__decode_order1])) * stbir__max_uint16_as_float_inverted;
9134+ decode[2-4] = ((float)(input[stbir__decode_order2])) * stbir__max_uint16_as_float_inverted;
9135+ decode[3-4] = ((float)(input[stbir__decode_order3])) * stbir__max_uint16_as_float_inverted;
9136+ decode += 4;
9137+ input += 4;
9138+ }
9139+ decode -= 4;
9140+ #endif
9141+
9142+ // do the remnants
9143+ #if stbir__coder_min_num < 4
9144+ STBIR_NO_UNROLL_LOOP_START
9145+ while( decode < decode_end )
9146+ {
9147+ STBIR_NO_UNROLL(decode);
9148+ decode[0] = ((float)(input[stbir__decode_order0])) * stbir__max_uint16_as_float_inverted;
9149+ #if stbir__coder_min_num >= 2
9150+ decode[1] = ((float)(input[stbir__decode_order1])) * stbir__max_uint16_as_float_inverted;
9151+ #endif
9152+ #if stbir__coder_min_num >= 3
9153+ decode[2] = ((float)(input[stbir__decode_order2])) * stbir__max_uint16_as_float_inverted;
9154+ #endif
9155+ decode += stbir__coder_min_num;
9156+ input += stbir__coder_min_num;
9157+ }
9158+ #endif
9159+ return decode_end;
9160+}
9161+
9162+
9163+static void STBIR__CODER_NAME(stbir__encode_uint16_linear_scaled)( void * outputp, int width_times_channels, float const * encode )
9164+{
9165+ unsigned short STBIR_SIMD_STREAMOUT_PTR( * ) output = (unsigned short*) outputp;
9166+ unsigned short * end_output = ( (unsigned short*) output ) + width_times_channels;
9167+
9168+ #ifdef STBIR_SIMD
9169+ {
9170+ if ( width_times_channels >= stbir__simdfX_float_count*2 )
9171+ {
9172+ float const * end_encode_m8 = encode + width_times_channels - stbir__simdfX_float_count*2;
9173+ end_output -= stbir__simdfX_float_count*2;
9174+ STBIR_SIMD_NO_UNROLL_LOOP_START_INF_FOR
9175+ for(;;)
9176+ {
9177+ stbir__simdfX e0, e1;
9178+ stbir__simdiX i;
9179+ STBIR_SIMD_NO_UNROLL(encode);
9180+ stbir__simdfX_madd_mem( e0, STBIR_simd_point5X, STBIR_max_uint16_as_floatX, encode );
9181+ stbir__simdfX_madd_mem( e1, STBIR_simd_point5X, STBIR_max_uint16_as_floatX, encode+stbir__simdfX_float_count );
9182+ stbir__encode_simdfX_unflip( e0 );
9183+ stbir__encode_simdfX_unflip( e1 );
9184+ stbir__simdfX_pack_to_words( i, e0, e1 );
9185+ stbir__simdiX_store( output, i );
9186+ encode += stbir__simdfX_float_count*2;
9187+ output += stbir__simdfX_float_count*2;
9188+ if ( output <= end_output )
9189+ continue;
9190+ if ( output == ( end_output + stbir__simdfX_float_count*2 ) )
9191+ break;
9192+ output = end_output; // backup and do last couple
9193+ encode = end_encode_m8;
9194+ }
9195+ return;
9196+ }
9197+ }
9198+
9199+ // try to do blocks of 4 when you can
9200+ #if stbir__coder_min_num != 3 // doesn't divide cleanly by four
9201+ output += 4;
9202+ STBIR_NO_UNROLL_LOOP_START
9203+ while( output <= end_output )
9204+ {
9205+ stbir__simdf e;
9206+ stbir__simdi i;
9207+ STBIR_NO_UNROLL(encode);
9208+ stbir__simdf_load( e, encode );
9209+ stbir__simdf_madd( e, STBIR__CONSTF(STBIR_simd_point5), STBIR__CONSTF(STBIR_max_uint16_as_float), e );
9210+ stbir__encode_simdf4_unflip( e );
9211+ stbir__simdf_pack_to_8words( i, e, e ); // only use first 4
9212+ stbir__simdi_store2( output-4, i );
9213+ output += 4;
9214+ encode += 4;
9215+ }
9216+ output -= 4;
9217+ #endif
9218+
9219+ // do the remnants
9220+ #if stbir__coder_min_num < 4
9221+ STBIR_NO_UNROLL_LOOP_START
9222+ while( output < end_output )
9223+ {
9224+ stbir__simdf e;
9225+ STBIR_NO_UNROLL(encode);
9226+ stbir__simdf_madd1_mem( e, STBIR__CONSTF(STBIR_simd_point5), STBIR__CONSTF(STBIR_max_uint16_as_float), encode+stbir__encode_order0 ); output[0] = stbir__simdf_convert_float_to_short( e );
9227+ #if stbir__coder_min_num >= 2
9228+ stbir__simdf_madd1_mem( e, STBIR__CONSTF(STBIR_simd_point5), STBIR__CONSTF(STBIR_max_uint16_as_float), encode+stbir__encode_order1 ); output[1] = stbir__simdf_convert_float_to_short( e );
9229+ #endif
9230+ #if stbir__coder_min_num >= 3
9231+ stbir__simdf_madd1_mem( e, STBIR__CONSTF(STBIR_simd_point5), STBIR__CONSTF(STBIR_max_uint16_as_float), encode+stbir__encode_order2 ); output[2] = stbir__simdf_convert_float_to_short( e );
9232+ #endif
9233+ output += stbir__coder_min_num;
9234+ encode += stbir__coder_min_num;
9235+ }
9236+ #endif
9237+
9238+ #else
9239+
9240+ // try to do blocks of 4 when you can
9241+ #if stbir__coder_min_num != 3 // doesn't divide cleanly by four
9242+ output += 4;
9243+ STBIR_SIMD_NO_UNROLL_LOOP_START
9244+ while( output <= end_output )
9245+ {
9246+ float f;
9247+ STBIR_SIMD_NO_UNROLL(encode);
9248+ f = encode[stbir__encode_order0] * stbir__max_uint16_as_float + 0.5f; STBIR_CLAMP(f, 0, 65535); output[0-4] = (unsigned short)f;
9249+ f = encode[stbir__encode_order1] * stbir__max_uint16_as_float + 0.5f; STBIR_CLAMP(f, 0, 65535); output[1-4] = (unsigned short)f;
9250+ f = encode[stbir__encode_order2] * stbir__max_uint16_as_float + 0.5f; STBIR_CLAMP(f, 0, 65535); output[2-4] = (unsigned short)f;
9251+ f = encode[stbir__encode_order3] * stbir__max_uint16_as_float + 0.5f; STBIR_CLAMP(f, 0, 65535); output[3-4] = (unsigned short)f;
9252+ output += 4;
9253+ encode += 4;
9254+ }
9255+ output -= 4;
9256+ #endif
9257+
9258+ // do the remnants
9259+ #if stbir__coder_min_num < 4
9260+ STBIR_NO_UNROLL_LOOP_START
9261+ while( output < end_output )
9262+ {
9263+ float f;
9264+ STBIR_NO_UNROLL(encode);
9265+ f = encode[stbir__encode_order0] * stbir__max_uint16_as_float + 0.5f; STBIR_CLAMP(f, 0, 65535); output[0] = (unsigned short)f;
9266+ #if stbir__coder_min_num >= 2
9267+ f = encode[stbir__encode_order1] * stbir__max_uint16_as_float + 0.5f; STBIR_CLAMP(f, 0, 65535); output[1] = (unsigned short)f;
9268+ #endif
9269+ #if stbir__coder_min_num >= 3
9270+ f = encode[stbir__encode_order2] * stbir__max_uint16_as_float + 0.5f; STBIR_CLAMP(f, 0, 65535); output[2] = (unsigned short)f;
9271+ #endif
9272+ output += stbir__coder_min_num;
9273+ encode += stbir__coder_min_num;
9274+ }
9275+ #endif
9276+ #endif
9277+}
9278+
9279+static float * STBIR__CODER_NAME(stbir__decode_uint16_linear)( float * decodep, int width_times_channels, void const * inputp )
9280+{
9281+ float STBIR_STREAMOUT_PTR( * ) decode = decodep;
9282+ float * decode_end = (float*) decode + width_times_channels;
9283+ unsigned short const * input = (unsigned short const *)inputp;
9284+
9285+ #ifdef STBIR_SIMD
9286+ unsigned short const * end_input_m8 = input + width_times_channels - 8;
9287+ if ( width_times_channels >= 8 )
9288+ {
9289+ decode_end -= 8;
9290+ STBIR_NO_UNROLL_LOOP_START_INF_FOR
9291+ for(;;)
9292+ {
9293+ #ifdef STBIR_SIMD8
9294+ stbir__simdi i; stbir__simdi8 o;
9295+ stbir__simdf8 of;
9296+ STBIR_NO_UNROLL(decode);
9297+ stbir__simdi_load( i, input );
9298+ stbir__simdi8_expand_u16_to_u32( o, i );
9299+ stbir__simdi8_convert_i32_to_float( of, o );
9300+ stbir__decode_simdf8_flip( of );
9301+ stbir__simdf8_store( decode + 0, of );
9302+ #else
9303+ stbir__simdi i, o0, o1;
9304+ stbir__simdf of0, of1;
9305+ STBIR_NO_UNROLL(decode);
9306+ stbir__simdi_load( i, input );
9307+ stbir__simdi_expand_u16_to_u32( o0, o1, i );
9308+ stbir__simdi_convert_i32_to_float( of0, o0 );
9309+ stbir__simdi_convert_i32_to_float( of1, o1 );
9310+ stbir__decode_simdf4_flip( of0 );
9311+ stbir__decode_simdf4_flip( of1 );
9312+ stbir__simdf_store( decode + 0, of0 );
9313+ stbir__simdf_store( decode + 4, of1 );
9314+ #endif
9315+ decode += 8;
9316+ input += 8;
9317+ if ( decode <= decode_end )
9318+ continue;
9319+ if ( decode == ( decode_end + 8 ) )
9320+ break;
9321+ decode = decode_end; // backup and do last couple
9322+ input = end_input_m8;
9323+ }
9324+ return decode_end + 8;
9325+ }
9326+ #endif
9327+
9328+ // try to do blocks of 4 when you can
9329+ #if stbir__coder_min_num != 3 // doesn't divide cleanly by four
9330+ decode += 4;
9331+ STBIR_SIMD_NO_UNROLL_LOOP_START
9332+ while( decode <= decode_end )
9333+ {
9334+ STBIR_SIMD_NO_UNROLL(decode);
9335+ decode[0-4] = ((float)(input[stbir__decode_order0]));
9336+ decode[1-4] = ((float)(input[stbir__decode_order1]));
9337+ decode[2-4] = ((float)(input[stbir__decode_order2]));
9338+ decode[3-4] = ((float)(input[stbir__decode_order3]));
9339+ decode += 4;
9340+ input += 4;
9341+ }
9342+ decode -= 4;
9343+ #endif
9344+
9345+ // do the remnants
9346+ #if stbir__coder_min_num < 4
9347+ STBIR_NO_UNROLL_LOOP_START
9348+ while( decode < decode_end )
9349+ {
9350+ STBIR_NO_UNROLL(decode);
9351+ decode[0] = ((float)(input[stbir__decode_order0]));
9352+ #if stbir__coder_min_num >= 2
9353+ decode[1] = ((float)(input[stbir__decode_order1]));
9354+ #endif
9355+ #if stbir__coder_min_num >= 3
9356+ decode[2] = ((float)(input[stbir__decode_order2]));
9357+ #endif
9358+ decode += stbir__coder_min_num;
9359+ input += stbir__coder_min_num;
9360+ }
9361+ #endif
9362+ return decode_end;
9363+}
9364+
9365+static void STBIR__CODER_NAME(stbir__encode_uint16_linear)( void * outputp, int width_times_channels, float const * encode )
9366+{
9367+ unsigned short STBIR_SIMD_STREAMOUT_PTR( * ) output = (unsigned short*) outputp;
9368+ unsigned short * end_output = ( (unsigned short*) output ) + width_times_channels;
9369+
9370+ #ifdef STBIR_SIMD
9371+ {
9372+ if ( width_times_channels >= stbir__simdfX_float_count*2 )
9373+ {
9374+ float const * end_encode_m8 = encode + width_times_channels - stbir__simdfX_float_count*2;
9375+ end_output -= stbir__simdfX_float_count*2;
9376+ STBIR_SIMD_NO_UNROLL_LOOP_START_INF_FOR
9377+ for(;;)
9378+ {
9379+ stbir__simdfX e0, e1;
9380+ stbir__simdiX i;
9381+ STBIR_SIMD_NO_UNROLL(encode);
9382+ stbir__simdfX_add_mem( e0, STBIR_simd_point5X, encode );
9383+ stbir__simdfX_add_mem( e1, STBIR_simd_point5X, encode+stbir__simdfX_float_count );
9384+ stbir__encode_simdfX_unflip( e0 );
9385+ stbir__encode_simdfX_unflip( e1 );
9386+ stbir__simdfX_pack_to_words( i, e0, e1 );
9387+ stbir__simdiX_store( output, i );
9388+ encode += stbir__simdfX_float_count*2;
9389+ output += stbir__simdfX_float_count*2;
9390+ if ( output <= end_output )
9391+ continue;
9392+ if ( output == ( end_output + stbir__simdfX_float_count*2 ) )
9393+ break;
9394+ output = end_output; // backup and do last couple
9395+ encode = end_encode_m8;
9396+ }
9397+ return;
9398+ }
9399+ }
9400+
9401+ // try to do blocks of 4 when you can
9402+ #if stbir__coder_min_num != 3 // doesn't divide cleanly by four
9403+ output += 4;
9404+ STBIR_NO_UNROLL_LOOP_START
9405+ while( output <= end_output )
9406+ {
9407+ stbir__simdf e;
9408+ stbir__simdi i;
9409+ STBIR_NO_UNROLL(encode);
9410+ stbir__simdf_load( e, encode );
9411+ stbir__simdf_add( e, STBIR__CONSTF(STBIR_simd_point5), e );
9412+ stbir__encode_simdf4_unflip( e );
9413+ stbir__simdf_pack_to_8words( i, e, e ); // only use first 4
9414+ stbir__simdi_store2( output-4, i );
9415+ output += 4;
9416+ encode += 4;
9417+ }
9418+ output -= 4;
9419+ #endif
9420+
9421+ #else
9422+
9423+ // try to do blocks of 4 when you can
9424+ #if stbir__coder_min_num != 3 // doesn't divide cleanly by four
9425+ output += 4;
9426+ STBIR_SIMD_NO_UNROLL_LOOP_START
9427+ while( output <= end_output )
9428+ {
9429+ float f;
9430+ STBIR_SIMD_NO_UNROLL(encode);
9431+ f = encode[stbir__encode_order0] + 0.5f; STBIR_CLAMP(f, 0, 65535); output[0-4] = (unsigned short)f;
9432+ f = encode[stbir__encode_order1] + 0.5f; STBIR_CLAMP(f, 0, 65535); output[1-4] = (unsigned short)f;
9433+ f = encode[stbir__encode_order2] + 0.5f; STBIR_CLAMP(f, 0, 65535); output[2-4] = (unsigned short)f;
9434+ f = encode[stbir__encode_order3] + 0.5f; STBIR_CLAMP(f, 0, 65535); output[3-4] = (unsigned short)f;
9435+ output += 4;
9436+ encode += 4;
9437+ }
9438+ output -= 4;
9439+ #endif
9440+
9441+ #endif
9442+
9443+ // do the remnants
9444+ #if stbir__coder_min_num < 4
9445+ STBIR_NO_UNROLL_LOOP_START
9446+ while( output < end_output )
9447+ {
9448+ float f;
9449+ STBIR_NO_UNROLL(encode);
9450+ f = encode[stbir__encode_order0] + 0.5f; STBIR_CLAMP(f, 0, 65535); output[0] = (unsigned short)f;
9451+ #if stbir__coder_min_num >= 2
9452+ f = encode[stbir__encode_order1] + 0.5f; STBIR_CLAMP(f, 0, 65535); output[1] = (unsigned short)f;
9453+ #endif
9454+ #if stbir__coder_min_num >= 3
9455+ f = encode[stbir__encode_order2] + 0.5f; STBIR_CLAMP(f, 0, 65535); output[2] = (unsigned short)f;
9456+ #endif
9457+ output += stbir__coder_min_num;
9458+ encode += stbir__coder_min_num;
9459+ }
9460+ #endif
9461+}
9462+
9463+static float * STBIR__CODER_NAME(stbir__decode_half_float_linear)( float * decodep, int width_times_channels, void const * inputp )
9464+{
9465+ float STBIR_STREAMOUT_PTR( * ) decode = decodep;
9466+ float * decode_end = (float*) decode + width_times_channels;
9467+ stbir__FP16 const * input = (stbir__FP16 const *)inputp;
9468+
9469+ #ifdef STBIR_SIMD
9470+ if ( width_times_channels >= 8 )
9471+ {
9472+ stbir__FP16 const * end_input_m8 = input + width_times_channels - 8;
9473+ decode_end -= 8;
9474+ STBIR_NO_UNROLL_LOOP_START_INF_FOR
9475+ for(;;)
9476+ {
9477+ STBIR_NO_UNROLL(decode);
9478+
9479+ stbir__half_to_float_SIMD( decode, input );
9480+ #ifdef stbir__decode_swizzle
9481+ #ifdef STBIR_SIMD8
9482+ {
9483+ stbir__simdf8 of;
9484+ stbir__simdf8_load( of, decode );
9485+ stbir__decode_simdf8_flip( of );
9486+ stbir__simdf8_store( decode, of );
9487+ }
9488+ #else
9489+ {
9490+ stbir__simdf of0,of1;
9491+ stbir__simdf_load( of0, decode );
9492+ stbir__simdf_load( of1, decode+4 );
9493+ stbir__decode_simdf4_flip( of0 );
9494+ stbir__decode_simdf4_flip( of1 );
9495+ stbir__simdf_store( decode, of0 );
9496+ stbir__simdf_store( decode+4, of1 );
9497+ }
9498+ #endif
9499+ #endif
9500+ decode += 8;
9501+ input += 8;
9502+ if ( decode <= decode_end )
9503+ continue;
9504+ if ( decode == ( decode_end + 8 ) )
9505+ break;
9506+ decode = decode_end; // backup and do last couple
9507+ input = end_input_m8;
9508+ }
9509+ return decode_end + 8;
9510+ }
9511+ #endif
9512+
9513+ // try to do blocks of 4 when you can
9514+ #if stbir__coder_min_num != 3 // doesn't divide cleanly by four
9515+ decode += 4;
9516+ STBIR_SIMD_NO_UNROLL_LOOP_START
9517+ while( decode <= decode_end )
9518+ {
9519+ STBIR_SIMD_NO_UNROLL(decode);
9520+ decode[0-4] = stbir__half_to_float(input[stbir__decode_order0]);
9521+ decode[1-4] = stbir__half_to_float(input[stbir__decode_order1]);
9522+ decode[2-4] = stbir__half_to_float(input[stbir__decode_order2]);
9523+ decode[3-4] = stbir__half_to_float(input[stbir__decode_order3]);
9524+ decode += 4;
9525+ input += 4;
9526+ }
9527+ decode -= 4;
9528+ #endif
9529+
9530+ // do the remnants
9531+ #if stbir__coder_min_num < 4
9532+ STBIR_NO_UNROLL_LOOP_START
9533+ while( decode < decode_end )
9534+ {
9535+ STBIR_NO_UNROLL(decode);
9536+ decode[0] = stbir__half_to_float(input[stbir__decode_order0]);
9537+ #if stbir__coder_min_num >= 2
9538+ decode[1] = stbir__half_to_float(input[stbir__decode_order1]);
9539+ #endif
9540+ #if stbir__coder_min_num >= 3
9541+ decode[2] = stbir__half_to_float(input[stbir__decode_order2]);
9542+ #endif
9543+ decode += stbir__coder_min_num;
9544+ input += stbir__coder_min_num;
9545+ }
9546+ #endif
9547+ return decode_end;
9548+}
9549+
9550+static void STBIR__CODER_NAME( stbir__encode_half_float_linear )( void * outputp, int width_times_channels, float const * encode )
9551+{
9552+ stbir__FP16 STBIR_SIMD_STREAMOUT_PTR( * ) output = (stbir__FP16*) outputp;
9553+ stbir__FP16 * end_output = ( (stbir__FP16*) output ) + width_times_channels;
9554+
9555+ #ifdef STBIR_SIMD
9556+ if ( width_times_channels >= 8 )
9557+ {
9558+ float const * end_encode_m8 = encode + width_times_channels - 8;
9559+ end_output -= 8;
9560+ STBIR_SIMD_NO_UNROLL_LOOP_START_INF_FOR
9561+ for(;;)
9562+ {
9563+ STBIR_SIMD_NO_UNROLL(encode);
9564+ #ifdef stbir__decode_swizzle
9565+ #ifdef STBIR_SIMD8
9566+ {
9567+ stbir__simdf8 of;
9568+ stbir__simdf8_load( of, encode );
9569+ stbir__encode_simdf8_unflip( of );
9570+ stbir__float_to_half_SIMD( output, (float*)&of );
9571+ }
9572+ #else
9573+ {
9574+ stbir__simdf of[2];
9575+ stbir__simdf_load( of[0], encode );
9576+ stbir__simdf_load( of[1], encode+4 );
9577+ stbir__encode_simdf4_unflip( of[0] );
9578+ stbir__encode_simdf4_unflip( of[1] );
9579+ stbir__float_to_half_SIMD( output, (float*)of );
9580+ }
9581+ #endif
9582+ #else
9583+ stbir__float_to_half_SIMD( output, encode );
9584+ #endif
9585+ encode += 8;
9586+ output += 8;
9587+ if ( output <= end_output )
9588+ continue;
9589+ if ( output == ( end_output + 8 ) )
9590+ break;
9591+ output = end_output; // backup and do last couple
9592+ encode = end_encode_m8;
9593+ }
9594+ return;
9595+ }
9596+ #endif
9597+
9598+ // try to do blocks of 4 when you can
9599+ #if stbir__coder_min_num != 3 // doesn't divide cleanly by four
9600+ output += 4;
9601+ STBIR_SIMD_NO_UNROLL_LOOP_START
9602+ while( output <= end_output )
9603+ {
9604+ STBIR_SIMD_NO_UNROLL(output);
9605+ output[0-4] = stbir__float_to_half(encode[stbir__encode_order0]);
9606+ output[1-4] = stbir__float_to_half(encode[stbir__encode_order1]);
9607+ output[2-4] = stbir__float_to_half(encode[stbir__encode_order2]);
9608+ output[3-4] = stbir__float_to_half(encode[stbir__encode_order3]);
9609+ output += 4;
9610+ encode += 4;
9611+ }
9612+ output -= 4;
9613+ #endif
9614+
9615+ // do the remnants
9616+ #if stbir__coder_min_num < 4
9617+ STBIR_NO_UNROLL_LOOP_START
9618+ while( output < end_output )
9619+ {
9620+ STBIR_NO_UNROLL(output);
9621+ output[0] = stbir__float_to_half(encode[stbir__encode_order0]);
9622+ #if stbir__coder_min_num >= 2
9623+ output[1] = stbir__float_to_half(encode[stbir__encode_order1]);
9624+ #endif
9625+ #if stbir__coder_min_num >= 3
9626+ output[2] = stbir__float_to_half(encode[stbir__encode_order2]);
9627+ #endif
9628+ output += stbir__coder_min_num;
9629+ encode += stbir__coder_min_num;
9630+ }
9631+ #endif
9632+}
9633+
9634+static float * STBIR__CODER_NAME(stbir__decode_float_linear)( float * decodep, int width_times_channels, void const * inputp )
9635+{
9636+ #ifdef stbir__decode_swizzle
9637+ float STBIR_STREAMOUT_PTR( * ) decode = decodep;
9638+ float * decode_end = (float*) decode + width_times_channels;
9639+ float const * input = (float const *)inputp;
9640+
9641+ #ifdef STBIR_SIMD
9642+ if ( width_times_channels >= 16 )
9643+ {
9644+ float const * end_input_m16 = input + width_times_channels - 16;
9645+ decode_end -= 16;
9646+ STBIR_NO_UNROLL_LOOP_START_INF_FOR
9647+ for(;;)
9648+ {
9649+ STBIR_NO_UNROLL(decode);
9650+ #ifdef stbir__decode_swizzle
9651+ #ifdef STBIR_SIMD8
9652+ {
9653+ stbir__simdf8 of0,of1;
9654+ stbir__simdf8_load( of0, input );
9655+ stbir__simdf8_load( of1, input+8 );
9656+ stbir__decode_simdf8_flip( of0 );
9657+ stbir__decode_simdf8_flip( of1 );
9658+ stbir__simdf8_store( decode, of0 );
9659+ stbir__simdf8_store( decode+8, of1 );
9660+ }
9661+ #else
9662+ {
9663+ stbir__simdf of0,of1,of2,of3;
9664+ stbir__simdf_load( of0, input );
9665+ stbir__simdf_load( of1, input+4 );
9666+ stbir__simdf_load( of2, input+8 );
9667+ stbir__simdf_load( of3, input+12 );
9668+ stbir__decode_simdf4_flip( of0 );
9669+ stbir__decode_simdf4_flip( of1 );
9670+ stbir__decode_simdf4_flip( of2 );
9671+ stbir__decode_simdf4_flip( of3 );
9672+ stbir__simdf_store( decode, of0 );
9673+ stbir__simdf_store( decode+4, of1 );
9674+ stbir__simdf_store( decode+8, of2 );
9675+ stbir__simdf_store( decode+12, of3 );
9676+ }
9677+ #endif
9678+ #endif
9679+ decode += 16;
9680+ input += 16;
9681+ if ( decode <= decode_end )
9682+ continue;
9683+ if ( decode == ( decode_end + 16 ) )
9684+ break;
9685+ decode = decode_end; // backup and do last couple
9686+ input = end_input_m16;
9687+ }
9688+ return decode_end + 16;
9689+ }
9690+ #endif
9691+
9692+ // try to do blocks of 4 when you can
9693+ #if stbir__coder_min_num != 3 // doesn't divide cleanly by four
9694+ decode += 4;
9695+ STBIR_SIMD_NO_UNROLL_LOOP_START
9696+ while( decode <= decode_end )
9697+ {
9698+ STBIR_SIMD_NO_UNROLL(decode);
9699+ decode[0-4] = input[stbir__decode_order0];
9700+ decode[1-4] = input[stbir__decode_order1];
9701+ decode[2-4] = input[stbir__decode_order2];
9702+ decode[3-4] = input[stbir__decode_order3];
9703+ decode += 4;
9704+ input += 4;
9705+ }
9706+ decode -= 4;
9707+ #endif
9708+
9709+ // do the remnants
9710+ #if stbir__coder_min_num < 4
9711+ STBIR_NO_UNROLL_LOOP_START
9712+ while( decode < decode_end )
9713+ {
9714+ STBIR_NO_UNROLL(decode);
9715+ decode[0] = input[stbir__decode_order0];
9716+ #if stbir__coder_min_num >= 2
9717+ decode[1] = input[stbir__decode_order1];
9718+ #endif
9719+ #if stbir__coder_min_num >= 3
9720+ decode[2] = input[stbir__decode_order2];
9721+ #endif
9722+ decode += stbir__coder_min_num;
9723+ input += stbir__coder_min_num;
9724+ }
9725+ #endif
9726+ return decode_end;
9727+
9728+ #else
9729+
9730+ if ( (void*)decodep != inputp )
9731+ STBIR_MEMCPY( decodep, inputp, width_times_channels * sizeof( float ) );
9732+
9733+ return decodep + width_times_channels;
9734+
9735+ #endif
9736+}
9737+
9738+static void STBIR__CODER_NAME( stbir__encode_float_linear )( void * outputp, int width_times_channels, float const * encode )
9739+{
9740+ #if !defined( STBIR_FLOAT_HIGH_CLAMP ) && !defined(STBIR_FLOAT_LOW_CLAMP) && !defined(stbir__decode_swizzle)
9741+
9742+ if ( (void*)outputp != (void*) encode )
9743+ STBIR_MEMCPY( outputp, encode, width_times_channels * sizeof( float ) );
9744+
9745+ #else
9746+
9747+ float STBIR_SIMD_STREAMOUT_PTR( * ) output = (float*) outputp;
9748+ float * end_output = ( (float*) output ) + width_times_channels;
9749+
9750+ #ifdef STBIR_FLOAT_HIGH_CLAMP
9751+ #define stbir_scalar_hi_clamp( v ) if ( v > STBIR_FLOAT_HIGH_CLAMP ) v = STBIR_FLOAT_HIGH_CLAMP;
9752+ #else
9753+ #define stbir_scalar_hi_clamp( v )
9754+ #endif
9755+ #ifdef STBIR_FLOAT_LOW_CLAMP
9756+ #define stbir_scalar_lo_clamp( v ) if ( v < STBIR_FLOAT_LOW_CLAMP ) v = STBIR_FLOAT_LOW_CLAMP;
9757+ #else
9758+ #define stbir_scalar_lo_clamp( v )
9759+ #endif
9760+
9761+ #ifdef STBIR_SIMD
9762+
9763+ #ifdef STBIR_FLOAT_HIGH_CLAMP
9764+ const stbir__simdfX high_clamp = stbir__simdf_frepX(STBIR_FLOAT_HIGH_CLAMP);
9765+ #endif
9766+ #ifdef STBIR_FLOAT_LOW_CLAMP
9767+ const stbir__simdfX low_clamp = stbir__simdf_frepX(STBIR_FLOAT_LOW_CLAMP);
9768+ #endif
9769+
9770+ if ( width_times_channels >= ( stbir__simdfX_float_count * 2 ) )
9771+ {
9772+ float const * end_encode_m8 = encode + width_times_channels - ( stbir__simdfX_float_count * 2 );
9773+ end_output -= ( stbir__simdfX_float_count * 2 );
9774+ STBIR_SIMD_NO_UNROLL_LOOP_START_INF_FOR
9775+ for(;;)
9776+ {
9777+ stbir__simdfX e0, e1;
9778+ STBIR_SIMD_NO_UNROLL(encode);
9779+ stbir__simdfX_load( e0, encode );
9780+ stbir__simdfX_load( e1, encode+stbir__simdfX_float_count );
9781+#ifdef STBIR_FLOAT_HIGH_CLAMP
9782+ stbir__simdfX_min( e0, e0, high_clamp );
9783+ stbir__simdfX_min( e1, e1, high_clamp );
9784+#endif
9785+#ifdef STBIR_FLOAT_LOW_CLAMP
9786+ stbir__simdfX_max( e0, e0, low_clamp );
9787+ stbir__simdfX_max( e1, e1, low_clamp );
9788+#endif
9789+ stbir__encode_simdfX_unflip( e0 );
9790+ stbir__encode_simdfX_unflip( e1 );
9791+ stbir__simdfX_store( output, e0 );
9792+ stbir__simdfX_store( output+stbir__simdfX_float_count, e1 );
9793+ encode += stbir__simdfX_float_count * 2;
9794+ output += stbir__simdfX_float_count * 2;
9795+ if ( output <= end_output )
9796+ continue;
9797+ if ( output == ( end_output + ( stbir__simdfX_float_count * 2 ) ) )
9798+ break;
9799+ output = end_output; // backup and do last couple
9800+ encode = end_encode_m8;
9801+ }
9802+ return;
9803+ }
9804+
9805+ // try to do blocks of 4 when you can
9806+ #if stbir__coder_min_num != 3 // doesn't divide cleanly by four
9807+ output += 4;
9808+ STBIR_NO_UNROLL_LOOP_START
9809+ while( output <= end_output )
9810+ {
9811+ stbir__simdf e0;
9812+ STBIR_NO_UNROLL(encode);
9813+ stbir__simdf_load( e0, encode );
9814+#ifdef STBIR_FLOAT_HIGH_CLAMP
9815+ stbir__simdf_min( e0, e0, high_clamp );
9816+#endif
9817+#ifdef STBIR_FLOAT_LOW_CLAMP
9818+ stbir__simdf_max( e0, e0, low_clamp );
9819+#endif
9820+ stbir__encode_simdf4_unflip( e0 );
9821+ stbir__simdf_store( output-4, e0 );
9822+ output += 4;
9823+ encode += 4;
9824+ }
9825+ output -= 4;
9826+ #endif
9827+
9828+ #else
9829+
9830+ // try to do blocks of 4 when you can
9831+ #if stbir__coder_min_num != 3 // doesn't divide cleanly by four
9832+ output += 4;
9833+ STBIR_SIMD_NO_UNROLL_LOOP_START
9834+ while( output <= end_output )
9835+ {
9836+ float e;
9837+ STBIR_SIMD_NO_UNROLL(encode);
9838+ e = encode[ stbir__encode_order0 ]; stbir_scalar_hi_clamp( e ); stbir_scalar_lo_clamp( e ); output[0-4] = e;
9839+ e = encode[ stbir__encode_order1 ]; stbir_scalar_hi_clamp( e ); stbir_scalar_lo_clamp( e ); output[1-4] = e;
9840+ e = encode[ stbir__encode_order2 ]; stbir_scalar_hi_clamp( e ); stbir_scalar_lo_clamp( e ); output[2-4] = e;
9841+ e = encode[ stbir__encode_order3 ]; stbir_scalar_hi_clamp( e ); stbir_scalar_lo_clamp( e ); output[3-4] = e;
9842+ output += 4;
9843+ encode += 4;
9844+ }
9845+ output -= 4;
9846+
9847+ #endif
9848+
9849+ #endif
9850+
9851+ // do the remnants
9852+ #if stbir__coder_min_num < 4
9853+ STBIR_NO_UNROLL_LOOP_START
9854+ while( output < end_output )
9855+ {
9856+ float e;
9857+ STBIR_NO_UNROLL(encode);
9858+ e = encode[ stbir__encode_order0 ]; stbir_scalar_hi_clamp( e ); stbir_scalar_lo_clamp( e ); output[0] = e;
9859+ #if stbir__coder_min_num >= 2
9860+ e = encode[ stbir__encode_order1 ]; stbir_scalar_hi_clamp( e ); stbir_scalar_lo_clamp( e ); output[1] = e;
9861+ #endif
9862+ #if stbir__coder_min_num >= 3
9863+ e = encode[ stbir__encode_order2 ]; stbir_scalar_hi_clamp( e ); stbir_scalar_lo_clamp( e ); output[2] = e;
9864+ #endif
9865+ output += stbir__coder_min_num;
9866+ encode += stbir__coder_min_num;
9867+ }
9868+ #endif
9869+
9870+ #endif
9871+}
9872+
9873+#undef stbir__decode_suffix
9874+#undef stbir__decode_simdf8_flip
9875+#undef stbir__decode_simdf4_flip
9876+#undef stbir__decode_order0
9877+#undef stbir__decode_order1
9878+#undef stbir__decode_order2
9879+#undef stbir__decode_order3
9880+#undef stbir__encode_order0
9881+#undef stbir__encode_order1
9882+#undef stbir__encode_order2
9883+#undef stbir__encode_order3
9884+#undef stbir__encode_simdf8_unflip
9885+#undef stbir__encode_simdf4_unflip
9886+#undef stbir__encode_simdfX_unflip
9887+#undef STBIR__CODER_NAME
9888+#undef stbir__coder_min_num
9889+#undef stbir__decode_swizzle
9890+#undef stbir_scalar_hi_clamp
9891+#undef stbir_scalar_lo_clamp
9892+#undef STB_IMAGE_RESIZE_DO_CODERS
9893+
9894+#elif defined( STB_IMAGE_RESIZE_DO_VERTICALS)
9895+
9896+#ifdef STB_IMAGE_RESIZE_VERTICAL_CONTINUE
9897+#define STBIR_chans( start, end ) STBIR_strs_join14(start,STBIR__vertical_channels,end,_cont)
9898+#else
9899+#define STBIR_chans( start, end ) STBIR_strs_join1(start,STBIR__vertical_channels,end)
9900+#endif
9901+
9902+#if STBIR__vertical_channels >= 1
9903+#define stbIF0( code ) code
9904+#else
9905+#define stbIF0( code )
9906+#endif
9907+#if STBIR__vertical_channels >= 2
9908+#define stbIF1( code ) code
9909+#else
9910+#define stbIF1( code )
9911+#endif
9912+#if STBIR__vertical_channels >= 3
9913+#define stbIF2( code ) code
9914+#else
9915+#define stbIF2( code )
9916+#endif
9917+#if STBIR__vertical_channels >= 4
9918+#define stbIF3( code ) code
9919+#else
9920+#define stbIF3( code )
9921+#endif
9922+#if STBIR__vertical_channels >= 5
9923+#define stbIF4( code ) code
9924+#else
9925+#define stbIF4( code )
9926+#endif
9927+#if STBIR__vertical_channels >= 6
9928+#define stbIF5( code ) code
9929+#else
9930+#define stbIF5( code )
9931+#endif
9932+#if STBIR__vertical_channels >= 7
9933+#define stbIF6( code ) code
9934+#else
9935+#define stbIF6( code )
9936+#endif
9937+#if STBIR__vertical_channels >= 8
9938+#define stbIF7( code ) code
9939+#else
9940+#define stbIF7( code )
9941+#endif
9942+
9943+static void STBIR_chans( stbir__vertical_scatter_with_,_coeffs)( float ** outputs, float const * vertical_coefficients, float const * input, float const * input_end )
9944+{
9945+ stbIF0( float STBIR_SIMD_STREAMOUT_PTR( * ) output0 = outputs[0]; float c0s = vertical_coefficients[0]; )
9946+ stbIF1( float STBIR_SIMD_STREAMOUT_PTR( * ) output1 = outputs[1]; float c1s = vertical_coefficients[1]; )
9947+ stbIF2( float STBIR_SIMD_STREAMOUT_PTR( * ) output2 = outputs[2]; float c2s = vertical_coefficients[2]; )
9948+ stbIF3( float STBIR_SIMD_STREAMOUT_PTR( * ) output3 = outputs[3]; float c3s = vertical_coefficients[3]; )
9949+ stbIF4( float STBIR_SIMD_STREAMOUT_PTR( * ) output4 = outputs[4]; float c4s = vertical_coefficients[4]; )
9950+ stbIF5( float STBIR_SIMD_STREAMOUT_PTR( * ) output5 = outputs[5]; float c5s = vertical_coefficients[5]; )
9951+ stbIF6( float STBIR_SIMD_STREAMOUT_PTR( * ) output6 = outputs[6]; float c6s = vertical_coefficients[6]; )
9952+ stbIF7( float STBIR_SIMD_STREAMOUT_PTR( * ) output7 = outputs[7]; float c7s = vertical_coefficients[7]; )
9953+
9954+ #ifdef STBIR_SIMD
9955+ {
9956+ stbIF0(stbir__simdfX c0 = stbir__simdf_frepX( c0s ); )
9957+ stbIF1(stbir__simdfX c1 = stbir__simdf_frepX( c1s ); )
9958+ stbIF2(stbir__simdfX c2 = stbir__simdf_frepX( c2s ); )
9959+ stbIF3(stbir__simdfX c3 = stbir__simdf_frepX( c3s ); )
9960+ stbIF4(stbir__simdfX c4 = stbir__simdf_frepX( c4s ); )
9961+ stbIF5(stbir__simdfX c5 = stbir__simdf_frepX( c5s ); )
9962+ stbIF6(stbir__simdfX c6 = stbir__simdf_frepX( c6s ); )
9963+ stbIF7(stbir__simdfX c7 = stbir__simdf_frepX( c7s ); )
9964+ STBIR_SIMD_NO_UNROLL_LOOP_START
9965+ while ( ( (char*)input_end - (char*) input ) >= (16*stbir__simdfX_float_count) )
9966+ {
9967+ stbir__simdfX o0, o1, o2, o3, r0, r1, r2, r3;
9968+ STBIR_SIMD_NO_UNROLL(output0);
9969+
9970+ stbir__simdfX_load( r0, input ); stbir__simdfX_load( r1, input+stbir__simdfX_float_count ); stbir__simdfX_load( r2, input+(2*stbir__simdfX_float_count) ); stbir__simdfX_load( r3, input+(3*stbir__simdfX_float_count) );
9971+
9972+ #ifdef STB_IMAGE_RESIZE_VERTICAL_CONTINUE
9973+ stbIF0( stbir__simdfX_load( o0, output0 ); stbir__simdfX_load( o1, output0+stbir__simdfX_float_count ); stbir__simdfX_load( o2, output0+(2*stbir__simdfX_float_count) ); stbir__simdfX_load( o3, output0+(3*stbir__simdfX_float_count) );
9974+ stbir__simdfX_madd( o0, o0, r0, c0 ); stbir__simdfX_madd( o1, o1, r1, c0 ); stbir__simdfX_madd( o2, o2, r2, c0 ); stbir__simdfX_madd( o3, o3, r3, c0 );
9975+ stbir__simdfX_store( output0, o0 ); stbir__simdfX_store( output0+stbir__simdfX_float_count, o1 ); stbir__simdfX_store( output0+(2*stbir__simdfX_float_count), o2 ); stbir__simdfX_store( output0+(3*stbir__simdfX_float_count), o3 ); )
9976+ stbIF1( stbir__simdfX_load( o0, output1 ); stbir__simdfX_load( o1, output1+stbir__simdfX_float_count ); stbir__simdfX_load( o2, output1+(2*stbir__simdfX_float_count) ); stbir__simdfX_load( o3, output1+(3*stbir__simdfX_float_count) );
9977+ stbir__simdfX_madd( o0, o0, r0, c1 ); stbir__simdfX_madd( o1, o1, r1, c1 ); stbir__simdfX_madd( o2, o2, r2, c1 ); stbir__simdfX_madd( o3, o3, r3, c1 );
9978+ stbir__simdfX_store( output1, o0 ); stbir__simdfX_store( output1+stbir__simdfX_float_count, o1 ); stbir__simdfX_store( output1+(2*stbir__simdfX_float_count), o2 ); stbir__simdfX_store( output1+(3*stbir__simdfX_float_count), o3 ); )
9979+ stbIF2( stbir__simdfX_load( o0, output2 ); stbir__simdfX_load( o1, output2+stbir__simdfX_float_count ); stbir__simdfX_load( o2, output2+(2*stbir__simdfX_float_count) ); stbir__simdfX_load( o3, output2+(3*stbir__simdfX_float_count) );
9980+ stbir__simdfX_madd( o0, o0, r0, c2 ); stbir__simdfX_madd( o1, o1, r1, c2 ); stbir__simdfX_madd( o2, o2, r2, c2 ); stbir__simdfX_madd( o3, o3, r3, c2 );
9981+ stbir__simdfX_store( output2, o0 ); stbir__simdfX_store( output2+stbir__simdfX_float_count, o1 ); stbir__simdfX_store( output2+(2*stbir__simdfX_float_count), o2 ); stbir__simdfX_store( output2+(3*stbir__simdfX_float_count), o3 ); )
9982+ stbIF3( stbir__simdfX_load( o0, output3 ); stbir__simdfX_load( o1, output3+stbir__simdfX_float_count ); stbir__simdfX_load( o2, output3+(2*stbir__simdfX_float_count) ); stbir__simdfX_load( o3, output3+(3*stbir__simdfX_float_count) );
9983+ stbir__simdfX_madd( o0, o0, r0, c3 ); stbir__simdfX_madd( o1, o1, r1, c3 ); stbir__simdfX_madd( o2, o2, r2, c3 ); stbir__simdfX_madd( o3, o3, r3, c3 );
9984+ stbir__simdfX_store( output3, o0 ); stbir__simdfX_store( output3+stbir__simdfX_float_count, o1 ); stbir__simdfX_store( output3+(2*stbir__simdfX_float_count), o2 ); stbir__simdfX_store( output3+(3*stbir__simdfX_float_count), o3 ); )
9985+ stbIF4( stbir__simdfX_load( o0, output4 ); stbir__simdfX_load( o1, output4+stbir__simdfX_float_count ); stbir__simdfX_load( o2, output4+(2*stbir__simdfX_float_count) ); stbir__simdfX_load( o3, output4+(3*stbir__simdfX_float_count) );
9986+ stbir__simdfX_madd( o0, o0, r0, c4 ); stbir__simdfX_madd( o1, o1, r1, c4 ); stbir__simdfX_madd( o2, o2, r2, c4 ); stbir__simdfX_madd( o3, o3, r3, c4 );
9987+ stbir__simdfX_store( output4, o0 ); stbir__simdfX_store( output4+stbir__simdfX_float_count, o1 ); stbir__simdfX_store( output4+(2*stbir__simdfX_float_count), o2 ); stbir__simdfX_store( output4+(3*stbir__simdfX_float_count), o3 ); )
9988+ stbIF5( stbir__simdfX_load( o0, output5 ); stbir__simdfX_load( o1, output5+stbir__simdfX_float_count ); stbir__simdfX_load( o2, output5+(2*stbir__simdfX_float_count)); stbir__simdfX_load( o3, output5+(3*stbir__simdfX_float_count) );
9989+ stbir__simdfX_madd( o0, o0, r0, c5 ); stbir__simdfX_madd( o1, o1, r1, c5 ); stbir__simdfX_madd( o2, o2, r2, c5 ); stbir__simdfX_madd( o3, o3, r3, c5 );
9990+ stbir__simdfX_store( output5, o0 ); stbir__simdfX_store( output5+stbir__simdfX_float_count, o1 ); stbir__simdfX_store( output5+(2*stbir__simdfX_float_count), o2 ); stbir__simdfX_store( output5+(3*stbir__simdfX_float_count), o3 ); )
9991+ stbIF6( stbir__simdfX_load( o0, output6 ); stbir__simdfX_load( o1, output6+stbir__simdfX_float_count ); stbir__simdfX_load( o2, output6+(2*stbir__simdfX_float_count) ); stbir__simdfX_load( o3, output6+(3*stbir__simdfX_float_count) );
9992+ stbir__simdfX_madd( o0, o0, r0, c6 ); stbir__simdfX_madd( o1, o1, r1, c6 ); stbir__simdfX_madd( o2, o2, r2, c6 ); stbir__simdfX_madd( o3, o3, r3, c6 );
9993+ stbir__simdfX_store( output6, o0 ); stbir__simdfX_store( output6+stbir__simdfX_float_count, o1 ); stbir__simdfX_store( output6+(2*stbir__simdfX_float_count), o2 ); stbir__simdfX_store( output6+(3*stbir__simdfX_float_count), o3 ); )
9994+ stbIF7( stbir__simdfX_load( o0, output7 ); stbir__simdfX_load( o1, output7+stbir__simdfX_float_count ); stbir__simdfX_load( o2, output7+(2*stbir__simdfX_float_count) ); stbir__simdfX_load( o3, output7+(3*stbir__simdfX_float_count) );
9995+ stbir__simdfX_madd( o0, o0, r0, c7 ); stbir__simdfX_madd( o1, o1, r1, c7 ); stbir__simdfX_madd( o2, o2, r2, c7 ); stbir__simdfX_madd( o3, o3, r3, c7 );
9996+ stbir__simdfX_store( output7, o0 ); stbir__simdfX_store( output7+stbir__simdfX_float_count, o1 ); stbir__simdfX_store( output7+(2*stbir__simdfX_float_count), o2 ); stbir__simdfX_store( output7+(3*stbir__simdfX_float_count), o3 ); )
9997+ #else
9998+ stbIF0( stbir__simdfX_mult( o0, r0, c0 ); stbir__simdfX_mult( o1, r1, c0 ); stbir__simdfX_mult( o2, r2, c0 ); stbir__simdfX_mult( o3, r3, c0 );
9999+ stbir__simdfX_store( output0, o0 ); stbir__simdfX_store( output0+stbir__simdfX_float_count, o1 ); stbir__simdfX_store( output0+(2*stbir__simdfX_float_count), o2 ); stbir__simdfX_store( output0+(3*stbir__simdfX_float_count), o3 ); )
10000+ stbIF1( stbir__simdfX_mult( o0, r0, c1 ); stbir__simdfX_mult( o1, r1, c1 ); stbir__simdfX_mult( o2, r2, c1 ); stbir__simdfX_mult( o3, r3, c1 );
10001+ stbir__simdfX_store( output1, o0 ); stbir__simdfX_store( output1+stbir__simdfX_float_count, o1 ); stbir__simdfX_store( output1+(2*stbir__simdfX_float_count), o2 ); stbir__simdfX_store( output1+(3*stbir__simdfX_float_count), o3 ); )
10002+ stbIF2( stbir__simdfX_mult( o0, r0, c2 ); stbir__simdfX_mult( o1, r1, c2 ); stbir__simdfX_mult( o2, r2, c2 ); stbir__simdfX_mult( o3, r3, c2 );
10003+ stbir__simdfX_store( output2, o0 ); stbir__simdfX_store( output2+stbir__simdfX_float_count, o1 ); stbir__simdfX_store( output2+(2*stbir__simdfX_float_count), o2 ); stbir__simdfX_store( output2+(3*stbir__simdfX_float_count), o3 ); )
10004+ stbIF3( stbir__simdfX_mult( o0, r0, c3 ); stbir__simdfX_mult( o1, r1, c3 ); stbir__simdfX_mult( o2, r2, c3 ); stbir__simdfX_mult( o3, r3, c3 );
10005+ stbir__simdfX_store( output3, o0 ); stbir__simdfX_store( output3+stbir__simdfX_float_count, o1 ); stbir__simdfX_store( output3+(2*stbir__simdfX_float_count), o2 ); stbir__simdfX_store( output3+(3*stbir__simdfX_float_count), o3 ); )
10006+ stbIF4( stbir__simdfX_mult( o0, r0, c4 ); stbir__simdfX_mult( o1, r1, c4 ); stbir__simdfX_mult( o2, r2, c4 ); stbir__simdfX_mult( o3, r3, c4 );
10007+ stbir__simdfX_store( output4, o0 ); stbir__simdfX_store( output4+stbir__simdfX_float_count, o1 ); stbir__simdfX_store( output4+(2*stbir__simdfX_float_count), o2 ); stbir__simdfX_store( output4+(3*stbir__simdfX_float_count), o3 ); )
10008+ stbIF5( stbir__simdfX_mult( o0, r0, c5 ); stbir__simdfX_mult( o1, r1, c5 ); stbir__simdfX_mult( o2, r2, c5 ); stbir__simdfX_mult( o3, r3, c5 );
10009+ stbir__simdfX_store( output5, o0 ); stbir__simdfX_store( output5+stbir__simdfX_float_count, o1 ); stbir__simdfX_store( output5+(2*stbir__simdfX_float_count), o2 ); stbir__simdfX_store( output5+(3*stbir__simdfX_float_count), o3 ); )
10010+ stbIF6( stbir__simdfX_mult( o0, r0, c6 ); stbir__simdfX_mult( o1, r1, c6 ); stbir__simdfX_mult( o2, r2, c6 ); stbir__simdfX_mult( o3, r3, c6 );
10011+ stbir__simdfX_store( output6, o0 ); stbir__simdfX_store( output6+stbir__simdfX_float_count, o1 ); stbir__simdfX_store( output6+(2*stbir__simdfX_float_count), o2 ); stbir__simdfX_store( output6+(3*stbir__simdfX_float_count), o3 ); )
10012+ stbIF7( stbir__simdfX_mult( o0, r0, c7 ); stbir__simdfX_mult( o1, r1, c7 ); stbir__simdfX_mult( o2, r2, c7 ); stbir__simdfX_mult( o3, r3, c7 );
10013+ stbir__simdfX_store( output7, o0 ); stbir__simdfX_store( output7+stbir__simdfX_float_count, o1 ); stbir__simdfX_store( output7+(2*stbir__simdfX_float_count), o2 ); stbir__simdfX_store( output7+(3*stbir__simdfX_float_count), o3 ); )
10014+ #endif
10015+
10016+ input += (4*stbir__simdfX_float_count);
10017+ stbIF0( output0 += (4*stbir__simdfX_float_count); ) stbIF1( output1 += (4*stbir__simdfX_float_count); ) stbIF2( output2 += (4*stbir__simdfX_float_count); ) stbIF3( output3 += (4*stbir__simdfX_float_count); ) stbIF4( output4 += (4*stbir__simdfX_float_count); ) stbIF5( output5 += (4*stbir__simdfX_float_count); ) stbIF6( output6 += (4*stbir__simdfX_float_count); ) stbIF7( output7 += (4*stbir__simdfX_float_count); )
10018+ }
10019+ STBIR_SIMD_NO_UNROLL_LOOP_START
10020+ while ( ( (char*)input_end - (char*) input ) >= 16 )
10021+ {
10022+ stbir__simdf o0, r0;
10023+ STBIR_SIMD_NO_UNROLL(output0);
10024+
10025+ stbir__simdf_load( r0, input );
10026+
10027+ #ifdef STB_IMAGE_RESIZE_VERTICAL_CONTINUE
10028+ stbIF0( stbir__simdf_load( o0, output0 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c0 ) ); stbir__simdf_store( output0, o0 ); )
10029+ stbIF1( stbir__simdf_load( o0, output1 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c1 ) ); stbir__simdf_store( output1, o0 ); )
10030+ stbIF2( stbir__simdf_load( o0, output2 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c2 ) ); stbir__simdf_store( output2, o0 ); )
10031+ stbIF3( stbir__simdf_load( o0, output3 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c3 ) ); stbir__simdf_store( output3, o0 ); )
10032+ stbIF4( stbir__simdf_load( o0, output4 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c4 ) ); stbir__simdf_store( output4, o0 ); )
10033+ stbIF5( stbir__simdf_load( o0, output5 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c5 ) ); stbir__simdf_store( output5, o0 ); )
10034+ stbIF6( stbir__simdf_load( o0, output6 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c6 ) ); stbir__simdf_store( output6, o0 ); )
10035+ stbIF7( stbir__simdf_load( o0, output7 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c7 ) ); stbir__simdf_store( output7, o0 ); )
10036+ #else
10037+ stbIF0( stbir__simdf_mult( o0, r0, stbir__if_simdf8_cast_to_simdf4( c0 ) ); stbir__simdf_store( output0, o0 ); )
10038+ stbIF1( stbir__simdf_mult( o0, r0, stbir__if_simdf8_cast_to_simdf4( c1 ) ); stbir__simdf_store( output1, o0 ); )
10039+ stbIF2( stbir__simdf_mult( o0, r0, stbir__if_simdf8_cast_to_simdf4( c2 ) ); stbir__simdf_store( output2, o0 ); )
10040+ stbIF3( stbir__simdf_mult( o0, r0, stbir__if_simdf8_cast_to_simdf4( c3 ) ); stbir__simdf_store( output3, o0 ); )
10041+ stbIF4( stbir__simdf_mult( o0, r0, stbir__if_simdf8_cast_to_simdf4( c4 ) ); stbir__simdf_store( output4, o0 ); )
10042+ stbIF5( stbir__simdf_mult( o0, r0, stbir__if_simdf8_cast_to_simdf4( c5 ) ); stbir__simdf_store( output5, o0 ); )
10043+ stbIF6( stbir__simdf_mult( o0, r0, stbir__if_simdf8_cast_to_simdf4( c6 ) ); stbir__simdf_store( output6, o0 ); )
10044+ stbIF7( stbir__simdf_mult( o0, r0, stbir__if_simdf8_cast_to_simdf4( c7 ) ); stbir__simdf_store( output7, o0 ); )
10045+ #endif
10046+
10047+ input += 4;
10048+ stbIF0( output0 += 4; ) stbIF1( output1 += 4; ) stbIF2( output2 += 4; ) stbIF3( output3 += 4; ) stbIF4( output4 += 4; ) stbIF5( output5 += 4; ) stbIF6( output6 += 4; ) stbIF7( output7 += 4; )
10049+ }
10050+ }
10051+ #else
10052+ STBIR_NO_UNROLL_LOOP_START
10053+ while ( ( (char*)input_end - (char*) input ) >= 16 )
10054+ {
10055+ float r0, r1, r2, r3;
10056+ STBIR_NO_UNROLL(input);
10057+
10058+ r0 = input[0], r1 = input[1], r2 = input[2], r3 = input[3];
10059+
10060+ #ifdef STB_IMAGE_RESIZE_VERTICAL_CONTINUE
10061+ stbIF0( output0[0] += ( r0 * c0s ); output0[1] += ( r1 * c0s ); output0[2] += ( r2 * c0s ); output0[3] += ( r3 * c0s ); )
10062+ stbIF1( output1[0] += ( r0 * c1s ); output1[1] += ( r1 * c1s ); output1[2] += ( r2 * c1s ); output1[3] += ( r3 * c1s ); )
10063+ stbIF2( output2[0] += ( r0 * c2s ); output2[1] += ( r1 * c2s ); output2[2] += ( r2 * c2s ); output2[3] += ( r3 * c2s ); )
10064+ stbIF3( output3[0] += ( r0 * c3s ); output3[1] += ( r1 * c3s ); output3[2] += ( r2 * c3s ); output3[3] += ( r3 * c3s ); )
10065+ stbIF4( output4[0] += ( r0 * c4s ); output4[1] += ( r1 * c4s ); output4[2] += ( r2 * c4s ); output4[3] += ( r3 * c4s ); )
10066+ stbIF5( output5[0] += ( r0 * c5s ); output5[1] += ( r1 * c5s ); output5[2] += ( r2 * c5s ); output5[3] += ( r3 * c5s ); )
10067+ stbIF6( output6[0] += ( r0 * c6s ); output6[1] += ( r1 * c6s ); output6[2] += ( r2 * c6s ); output6[3] += ( r3 * c6s ); )
10068+ stbIF7( output7[0] += ( r0 * c7s ); output7[1] += ( r1 * c7s ); output7[2] += ( r2 * c7s ); output7[3] += ( r3 * c7s ); )
10069+ #else
10070+ stbIF0( output0[0] = ( r0 * c0s ); output0[1] = ( r1 * c0s ); output0[2] = ( r2 * c0s ); output0[3] = ( r3 * c0s ); )
10071+ stbIF1( output1[0] = ( r0 * c1s ); output1[1] = ( r1 * c1s ); output1[2] = ( r2 * c1s ); output1[3] = ( r3 * c1s ); )
10072+ stbIF2( output2[0] = ( r0 * c2s ); output2[1] = ( r1 * c2s ); output2[2] = ( r2 * c2s ); output2[3] = ( r3 * c2s ); )
10073+ stbIF3( output3[0] = ( r0 * c3s ); output3[1] = ( r1 * c3s ); output3[2] = ( r2 * c3s ); output3[3] = ( r3 * c3s ); )
10074+ stbIF4( output4[0] = ( r0 * c4s ); output4[1] = ( r1 * c4s ); output4[2] = ( r2 * c4s ); output4[3] = ( r3 * c4s ); )
10075+ stbIF5( output5[0] = ( r0 * c5s ); output5[1] = ( r1 * c5s ); output5[2] = ( r2 * c5s ); output5[3] = ( r3 * c5s ); )
10076+ stbIF6( output6[0] = ( r0 * c6s ); output6[1] = ( r1 * c6s ); output6[2] = ( r2 * c6s ); output6[3] = ( r3 * c6s ); )
10077+ stbIF7( output7[0] = ( r0 * c7s ); output7[1] = ( r1 * c7s ); output7[2] = ( r2 * c7s ); output7[3] = ( r3 * c7s ); )
10078+ #endif
10079+
10080+ input += 4;
10081+ stbIF0( output0 += 4; ) stbIF1( output1 += 4; ) stbIF2( output2 += 4; ) stbIF3( output3 += 4; ) stbIF4( output4 += 4; ) stbIF5( output5 += 4; ) stbIF6( output6 += 4; ) stbIF7( output7 += 4; )
10082+ }
10083+ #endif
10084+ STBIR_NO_UNROLL_LOOP_START
10085+ while ( input < input_end )
10086+ {
10087+ float r = input[0];
10088+ STBIR_NO_UNROLL(output0);
10089+
10090+ #ifdef STB_IMAGE_RESIZE_VERTICAL_CONTINUE
10091+ stbIF0( output0[0] += ( r * c0s ); )
10092+ stbIF1( output1[0] += ( r * c1s ); )
10093+ stbIF2( output2[0] += ( r * c2s ); )
10094+ stbIF3( output3[0] += ( r * c3s ); )
10095+ stbIF4( output4[0] += ( r * c4s ); )
10096+ stbIF5( output5[0] += ( r * c5s ); )
10097+ stbIF6( output6[0] += ( r * c6s ); )
10098+ stbIF7( output7[0] += ( r * c7s ); )
10099+ #else
10100+ stbIF0( output0[0] = ( r * c0s ); )
10101+ stbIF1( output1[0] = ( r * c1s ); )
10102+ stbIF2( output2[0] = ( r * c2s ); )
10103+ stbIF3( output3[0] = ( r * c3s ); )
10104+ stbIF4( output4[0] = ( r * c4s ); )
10105+ stbIF5( output5[0] = ( r * c5s ); )
10106+ stbIF6( output6[0] = ( r * c6s ); )
10107+ stbIF7( output7[0] = ( r * c7s ); )
10108+ #endif
10109+
10110+ ++input;
10111+ stbIF0( ++output0; ) stbIF1( ++output1; ) stbIF2( ++output2; ) stbIF3( ++output3; ) stbIF4( ++output4; ) stbIF5( ++output5; ) stbIF6( ++output6; ) stbIF7( ++output7; )
10112+ }
10113+}
10114+
10115+static void STBIR_chans( stbir__vertical_gather_with_,_coeffs)( float * outputp, float const * vertical_coefficients, float const ** inputs, float const * input0_end )
10116+{
10117+ float STBIR_SIMD_STREAMOUT_PTR( * ) output = outputp;
10118+
10119+ stbIF0( float const * input0 = inputs[0]; float c0s = vertical_coefficients[0]; )
10120+ stbIF1( float const * input1 = inputs[1]; float c1s = vertical_coefficients[1]; )
10121+ stbIF2( float const * input2 = inputs[2]; float c2s = vertical_coefficients[2]; )
10122+ stbIF3( float const * input3 = inputs[3]; float c3s = vertical_coefficients[3]; )
10123+ stbIF4( float const * input4 = inputs[4]; float c4s = vertical_coefficients[4]; )
10124+ stbIF5( float const * input5 = inputs[5]; float c5s = vertical_coefficients[5]; )
10125+ stbIF6( float const * input6 = inputs[6]; float c6s = vertical_coefficients[6]; )
10126+ stbIF7( float const * input7 = inputs[7]; float c7s = vertical_coefficients[7]; )
10127+
10128+#if ( STBIR__vertical_channels == 1 ) && !defined(STB_IMAGE_RESIZE_VERTICAL_CONTINUE)
10129+ // check single channel one weight
10130+ if ( ( c0s >= (1.0f-0.000001f) ) && ( c0s <= (1.0f+0.000001f) ) )
10131+ {
10132+ STBIR_MEMCPY( output, input0, (char*)input0_end - (char*)input0 );
10133+ return;
10134+ }
10135+#endif
10136+
10137+ #ifdef STBIR_SIMD
10138+ {
10139+ stbIF0(stbir__simdfX c0 = stbir__simdf_frepX( c0s ); )
10140+ stbIF1(stbir__simdfX c1 = stbir__simdf_frepX( c1s ); )
10141+ stbIF2(stbir__simdfX c2 = stbir__simdf_frepX( c2s ); )
10142+ stbIF3(stbir__simdfX c3 = stbir__simdf_frepX( c3s ); )
10143+ stbIF4(stbir__simdfX c4 = stbir__simdf_frepX( c4s ); )
10144+ stbIF5(stbir__simdfX c5 = stbir__simdf_frepX( c5s ); )
10145+ stbIF6(stbir__simdfX c6 = stbir__simdf_frepX( c6s ); )
10146+ stbIF7(stbir__simdfX c7 = stbir__simdf_frepX( c7s ); )
10147+
10148+ STBIR_SIMD_NO_UNROLL_LOOP_START
10149+ while ( ( (char*)input0_end - (char*) input0 ) >= (16*stbir__simdfX_float_count) )
10150+ {
10151+ stbir__simdfX o0, o1, o2, o3, r0, r1, r2, r3;
10152+ STBIR_SIMD_NO_UNROLL(output);
10153+
10154+ // prefetch four loop iterations ahead (doesn't affect much for small resizes, but helps with big ones)
10155+ stbIF0( stbir__prefetch( input0 + (16*stbir__simdfX_float_count) ); )
10156+ stbIF1( stbir__prefetch( input1 + (16*stbir__simdfX_float_count) ); )
10157+ stbIF2( stbir__prefetch( input2 + (16*stbir__simdfX_float_count) ); )
10158+ stbIF3( stbir__prefetch( input3 + (16*stbir__simdfX_float_count) ); )
10159+ stbIF4( stbir__prefetch( input4 + (16*stbir__simdfX_float_count) ); )
10160+ stbIF5( stbir__prefetch( input5 + (16*stbir__simdfX_float_count) ); )
10161+ stbIF6( stbir__prefetch( input6 + (16*stbir__simdfX_float_count) ); )
10162+ stbIF7( stbir__prefetch( input7 + (16*stbir__simdfX_float_count) ); )
10163+
10164+ #ifdef STB_IMAGE_RESIZE_VERTICAL_CONTINUE
10165+ stbIF0( stbir__simdfX_load( o0, output ); stbir__simdfX_load( o1, output+stbir__simdfX_float_count ); stbir__simdfX_load( o2, output+(2*stbir__simdfX_float_count) ); stbir__simdfX_load( o3, output+(3*stbir__simdfX_float_count) );
10166+ stbir__simdfX_load( r0, input0 ); stbir__simdfX_load( r1, input0+stbir__simdfX_float_count ); stbir__simdfX_load( r2, input0+(2*stbir__simdfX_float_count) ); stbir__simdfX_load( r3, input0+(3*stbir__simdfX_float_count) );
10167+ stbir__simdfX_madd( o0, o0, r0, c0 ); stbir__simdfX_madd( o1, o1, r1, c0 ); stbir__simdfX_madd( o2, o2, r2, c0 ); stbir__simdfX_madd( o3, o3, r3, c0 ); )
10168+ #else
10169+ stbIF0( stbir__simdfX_load( r0, input0 ); stbir__simdfX_load( r1, input0+stbir__simdfX_float_count ); stbir__simdfX_load( r2, input0+(2*stbir__simdfX_float_count) ); stbir__simdfX_load( r3, input0+(3*stbir__simdfX_float_count) );
10170+ stbir__simdfX_mult( o0, r0, c0 ); stbir__simdfX_mult( o1, r1, c0 ); stbir__simdfX_mult( o2, r2, c0 ); stbir__simdfX_mult( o3, r3, c0 ); )
10171+ #endif
10172+
10173+ stbIF1( stbir__simdfX_load( r0, input1 ); stbir__simdfX_load( r1, input1+stbir__simdfX_float_count ); stbir__simdfX_load( r2, input1+(2*stbir__simdfX_float_count) ); stbir__simdfX_load( r3, input1+(3*stbir__simdfX_float_count) );
10174+ stbir__simdfX_madd( o0, o0, r0, c1 ); stbir__simdfX_madd( o1, o1, r1, c1 ); stbir__simdfX_madd( o2, o2, r2, c1 ); stbir__simdfX_madd( o3, o3, r3, c1 ); )
10175+ stbIF2( stbir__simdfX_load( r0, input2 ); stbir__simdfX_load( r1, input2+stbir__simdfX_float_count ); stbir__simdfX_load( r2, input2+(2*stbir__simdfX_float_count) ); stbir__simdfX_load( r3, input2+(3*stbir__simdfX_float_count) );
10176+ stbir__simdfX_madd( o0, o0, r0, c2 ); stbir__simdfX_madd( o1, o1, r1, c2 ); stbir__simdfX_madd( o2, o2, r2, c2 ); stbir__simdfX_madd( o3, o3, r3, c2 ); )
10177+ stbIF3( stbir__simdfX_load( r0, input3 ); stbir__simdfX_load( r1, input3+stbir__simdfX_float_count ); stbir__simdfX_load( r2, input3+(2*stbir__simdfX_float_count) ); stbir__simdfX_load( r3, input3+(3*stbir__simdfX_float_count) );
10178+ stbir__simdfX_madd( o0, o0, r0, c3 ); stbir__simdfX_madd( o1, o1, r1, c3 ); stbir__simdfX_madd( o2, o2, r2, c3 ); stbir__simdfX_madd( o3, o3, r3, c3 ); )
10179+ stbIF4( stbir__simdfX_load( r0, input4 ); stbir__simdfX_load( r1, input4+stbir__simdfX_float_count ); stbir__simdfX_load( r2, input4+(2*stbir__simdfX_float_count) ); stbir__simdfX_load( r3, input4+(3*stbir__simdfX_float_count) );
10180+ stbir__simdfX_madd( o0, o0, r0, c4 ); stbir__simdfX_madd( o1, o1, r1, c4 ); stbir__simdfX_madd( o2, o2, r2, c4 ); stbir__simdfX_madd( o3, o3, r3, c4 ); )
10181+ stbIF5( stbir__simdfX_load( r0, input5 ); stbir__simdfX_load( r1, input5+stbir__simdfX_float_count ); stbir__simdfX_load( r2, input5+(2*stbir__simdfX_float_count) ); stbir__simdfX_load( r3, input5+(3*stbir__simdfX_float_count) );
10182+ stbir__simdfX_madd( o0, o0, r0, c5 ); stbir__simdfX_madd( o1, o1, r1, c5 ); stbir__simdfX_madd( o2, o2, r2, c5 ); stbir__simdfX_madd( o3, o3, r3, c5 ); )
10183+ stbIF6( stbir__simdfX_load( r0, input6 ); stbir__simdfX_load( r1, input6+stbir__simdfX_float_count ); stbir__simdfX_load( r2, input6+(2*stbir__simdfX_float_count) ); stbir__simdfX_load( r3, input6+(3*stbir__simdfX_float_count) );
10184+ stbir__simdfX_madd( o0, o0, r0, c6 ); stbir__simdfX_madd( o1, o1, r1, c6 ); stbir__simdfX_madd( o2, o2, r2, c6 ); stbir__simdfX_madd( o3, o3, r3, c6 ); )
10185+ stbIF7( stbir__simdfX_load( r0, input7 ); stbir__simdfX_load( r1, input7+stbir__simdfX_float_count ); stbir__simdfX_load( r2, input7+(2*stbir__simdfX_float_count) ); stbir__simdfX_load( r3, input7+(3*stbir__simdfX_float_count) );
10186+ stbir__simdfX_madd( o0, o0, r0, c7 ); stbir__simdfX_madd( o1, o1, r1, c7 ); stbir__simdfX_madd( o2, o2, r2, c7 ); stbir__simdfX_madd( o3, o3, r3, c7 ); )
10187+
10188+ stbir__simdfX_store( output, o0 ); stbir__simdfX_store( output+stbir__simdfX_float_count, o1 ); stbir__simdfX_store( output+(2*stbir__simdfX_float_count), o2 ); stbir__simdfX_store( output+(3*stbir__simdfX_float_count), o3 );
10189+ output += (4*stbir__simdfX_float_count);
10190+ stbIF0( input0 += (4*stbir__simdfX_float_count); ) stbIF1( input1 += (4*stbir__simdfX_float_count); ) stbIF2( input2 += (4*stbir__simdfX_float_count); ) stbIF3( input3 += (4*stbir__simdfX_float_count); ) stbIF4( input4 += (4*stbir__simdfX_float_count); ) stbIF5( input5 += (4*stbir__simdfX_float_count); ) stbIF6( input6 += (4*stbir__simdfX_float_count); ) stbIF7( input7 += (4*stbir__simdfX_float_count); )
10191+ }
10192+
10193+ STBIR_SIMD_NO_UNROLL_LOOP_START
10194+ while ( ( (char*)input0_end - (char*) input0 ) >= 16 )
10195+ {
10196+ stbir__simdf o0, r0;
10197+ STBIR_SIMD_NO_UNROLL(output);
10198+
10199+ #ifdef STB_IMAGE_RESIZE_VERTICAL_CONTINUE
10200+ stbIF0( stbir__simdf_load( o0, output ); stbir__simdf_load( r0, input0 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c0 ) ); )
10201+ #else
10202+ stbIF0( stbir__simdf_load( r0, input0 ); stbir__simdf_mult( o0, r0, stbir__if_simdf8_cast_to_simdf4( c0 ) ); )
10203+ #endif
10204+ stbIF1( stbir__simdf_load( r0, input1 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c1 ) ); )
10205+ stbIF2( stbir__simdf_load( r0, input2 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c2 ) ); )
10206+ stbIF3( stbir__simdf_load( r0, input3 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c3 ) ); )
10207+ stbIF4( stbir__simdf_load( r0, input4 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c4 ) ); )
10208+ stbIF5( stbir__simdf_load( r0, input5 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c5 ) ); )
10209+ stbIF6( stbir__simdf_load( r0, input6 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c6 ) ); )
10210+ stbIF7( stbir__simdf_load( r0, input7 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c7 ) ); )
10211+
10212+ stbir__simdf_store( output, o0 );
10213+ output += 4;
10214+ stbIF0( input0 += 4; ) stbIF1( input1 += 4; ) stbIF2( input2 += 4; ) stbIF3( input3 += 4; ) stbIF4( input4 += 4; ) stbIF5( input5 += 4; ) stbIF6( input6 += 4; ) stbIF7( input7 += 4; )
10215+ }
10216+ }
10217+ #else
10218+ STBIR_NO_UNROLL_LOOP_START
10219+ while ( ( (char*)input0_end - (char*) input0 ) >= 16 )
10220+ {
10221+ float o0, o1, o2, o3;
10222+ STBIR_NO_UNROLL(output);
10223+ #ifdef STB_IMAGE_RESIZE_VERTICAL_CONTINUE
10224+ stbIF0( o0 = output[0] + input0[0] * c0s; o1 = output[1] + input0[1] * c0s; o2 = output[2] + input0[2] * c0s; o3 = output[3] + input0[3] * c0s; )
10225+ #else
10226+ stbIF0( o0 = input0[0] * c0s; o1 = input0[1] * c0s; o2 = input0[2] * c0s; o3 = input0[3] * c0s; )
10227+ #endif
10228+ stbIF1( o0 += input1[0] * c1s; o1 += input1[1] * c1s; o2 += input1[2] * c1s; o3 += input1[3] * c1s; )
10229+ stbIF2( o0 += input2[0] * c2s; o1 += input2[1] * c2s; o2 += input2[2] * c2s; o3 += input2[3] * c2s; )
10230+ stbIF3( o0 += input3[0] * c3s; o1 += input3[1] * c3s; o2 += input3[2] * c3s; o3 += input3[3] * c3s; )
10231+ stbIF4( o0 += input4[0] * c4s; o1 += input4[1] * c4s; o2 += input4[2] * c4s; o3 += input4[3] * c4s; )
10232+ stbIF5( o0 += input5[0] * c5s; o1 += input5[1] * c5s; o2 += input5[2] * c5s; o3 += input5[3] * c5s; )
10233+ stbIF6( o0 += input6[0] * c6s; o1 += input6[1] * c6s; o2 += input6[2] * c6s; o3 += input6[3] * c6s; )
10234+ stbIF7( o0 += input7[0] * c7s; o1 += input7[1] * c7s; o2 += input7[2] * c7s; o3 += input7[3] * c7s; )
10235+ output[0] = o0; output[1] = o1; output[2] = o2; output[3] = o3;
10236+ output += 4;
10237+ stbIF0( input0 += 4; ) stbIF1( input1 += 4; ) stbIF2( input2 += 4; ) stbIF3( input3 += 4; ) stbIF4( input4 += 4; ) stbIF5( input5 += 4; ) stbIF6( input6 += 4; ) stbIF7( input7 += 4; )
10238+ }
10239+ #endif
10240+ STBIR_NO_UNROLL_LOOP_START
10241+ while ( input0 < input0_end )
10242+ {
10243+ float o0;
10244+ STBIR_NO_UNROLL(output);
10245+ #ifdef STB_IMAGE_RESIZE_VERTICAL_CONTINUE
10246+ stbIF0( o0 = output[0] + input0[0] * c0s; )
10247+ #else
10248+ stbIF0( o0 = input0[0] * c0s; )
10249+ #endif
10250+ stbIF1( o0 += input1[0] * c1s; )
10251+ stbIF2( o0 += input2[0] * c2s; )
10252+ stbIF3( o0 += input3[0] * c3s; )
10253+ stbIF4( o0 += input4[0] * c4s; )
10254+ stbIF5( o0 += input5[0] * c5s; )
10255+ stbIF6( o0 += input6[0] * c6s; )
10256+ stbIF7( o0 += input7[0] * c7s; )
10257+ output[0] = o0;
10258+ ++output;
10259+ stbIF0( ++input0; ) stbIF1( ++input1; ) stbIF2( ++input2; ) stbIF3( ++input3; ) stbIF4( ++input4; ) stbIF5( ++input5; ) stbIF6( ++input6; ) stbIF7( ++input7; )
10260+ }
10261+}
10262+
10263+#undef stbIF0
10264+#undef stbIF1
10265+#undef stbIF2
10266+#undef stbIF3
10267+#undef stbIF4
10268+#undef stbIF5
10269+#undef stbIF6
10270+#undef stbIF7
10271+#undef STB_IMAGE_RESIZE_DO_VERTICALS
10272+#undef STBIR__vertical_channels
10273+#undef STB_IMAGE_RESIZE_DO_HORIZONTALS
10274+#undef STBIR_strs_join24
10275+#undef STBIR_strs_join14
10276+#undef STBIR_chans
10277+#ifdef STB_IMAGE_RESIZE_VERTICAL_CONTINUE
10278+#undef STB_IMAGE_RESIZE_VERTICAL_CONTINUE
10279+#endif
10280+
10281+#else // !STB_IMAGE_RESIZE_DO_VERTICALS
10282+
10283+#define STBIR_chans( start, end ) STBIR_strs_join1(start,STBIR__horizontal_channels,end)
10284+
10285+#ifndef stbir__2_coeff_only
10286+#define stbir__2_coeff_only() \
10287+ stbir__1_coeff_only(); \
10288+ stbir__1_coeff_remnant(1);
10289+#endif
10290+
10291+#ifndef stbir__2_coeff_remnant
10292+#define stbir__2_coeff_remnant( ofs ) \
10293+ stbir__1_coeff_remnant(ofs); \
10294+ stbir__1_coeff_remnant((ofs)+1);
10295+#endif
10296+
10297+#ifndef stbir__3_coeff_only
10298+#define stbir__3_coeff_only() \
10299+ stbir__2_coeff_only(); \
10300+ stbir__1_coeff_remnant(2);
10301+#endif
10302+
10303+#ifndef stbir__3_coeff_remnant
10304+#define stbir__3_coeff_remnant( ofs ) \
10305+ stbir__2_coeff_remnant(ofs); \
10306+ stbir__1_coeff_remnant((ofs)+2);
10307+#endif
10308+
10309+#ifndef stbir__3_coeff_setup
10310+#define stbir__3_coeff_setup()
10311+#endif
10312+
10313+#ifndef stbir__4_coeff_start
10314+#define stbir__4_coeff_start() \
10315+ stbir__2_coeff_only(); \
10316+ stbir__2_coeff_remnant(2);
10317+#endif
10318+
10319+#ifndef stbir__4_coeff_continue_from_4
10320+#define stbir__4_coeff_continue_from_4( ofs ) \
10321+ stbir__2_coeff_remnant(ofs); \
10322+ stbir__2_coeff_remnant((ofs)+2);
10323+#endif
10324+
10325+#ifndef stbir__store_output_tiny
10326+#define stbir__store_output_tiny stbir__store_output
10327+#endif
10328+
10329+static void STBIR_chans( stbir__horizontal_gather_,_channels_with_1_coeff)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width )
10330+{
10331+ float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels;
10332+ float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer;
10333+ STBIR_SIMD_NO_UNROLL_LOOP_START
10334+ do {
10335+ float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels;
10336+ float const * hc = horizontal_coefficients;
10337+ stbir__1_coeff_only();
10338+ stbir__store_output_tiny();
10339+ } while ( output < output_end );
10340+}
10341+
10342+static void STBIR_chans( stbir__horizontal_gather_,_channels_with_2_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width )
10343+{
10344+ float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels;
10345+ float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer;
10346+ STBIR_SIMD_NO_UNROLL_LOOP_START
10347+ do {
10348+ float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels;
10349+ float const * hc = horizontal_coefficients;
10350+ stbir__2_coeff_only();
10351+ stbir__store_output_tiny();
10352+ } while ( output < output_end );
10353+}
10354+
10355+static void STBIR_chans( stbir__horizontal_gather_,_channels_with_3_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width )
10356+{
10357+ float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels;
10358+ float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer;
10359+ STBIR_SIMD_NO_UNROLL_LOOP_START
10360+ do {
10361+ float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels;
10362+ float const * hc = horizontal_coefficients;
10363+ stbir__3_coeff_only();
10364+ stbir__store_output_tiny();
10365+ } while ( output < output_end );
10366+}
10367+
10368+static void STBIR_chans( stbir__horizontal_gather_,_channels_with_4_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width )
10369+{
10370+ float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels;
10371+ float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer;
10372+ STBIR_SIMD_NO_UNROLL_LOOP_START
10373+ do {
10374+ float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels;
10375+ float const * hc = horizontal_coefficients;
10376+ stbir__4_coeff_start();
10377+ stbir__store_output();
10378+ } while ( output < output_end );
10379+}
10380+
10381+static void STBIR_chans( stbir__horizontal_gather_,_channels_with_5_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width )
10382+{
10383+ float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels;
10384+ float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer;
10385+ STBIR_SIMD_NO_UNROLL_LOOP_START
10386+ do {
10387+ float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels;
10388+ float const * hc = horizontal_coefficients;
10389+ stbir__4_coeff_start();
10390+ stbir__1_coeff_remnant(4);
10391+ stbir__store_output();
10392+ } while ( output < output_end );
10393+}
10394+
10395+static void STBIR_chans( stbir__horizontal_gather_,_channels_with_6_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width )
10396+{
10397+ float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels;
10398+ float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer;
10399+ STBIR_SIMD_NO_UNROLL_LOOP_START
10400+ do {
10401+ float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels;
10402+ float const * hc = horizontal_coefficients;
10403+ stbir__4_coeff_start();
10404+ stbir__2_coeff_remnant(4);
10405+ stbir__store_output();
10406+ } while ( output < output_end );
10407+}
10408+
10409+static void STBIR_chans( stbir__horizontal_gather_,_channels_with_7_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width )
10410+{
10411+ float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels;
10412+ float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer;
10413+ stbir__3_coeff_setup();
10414+ STBIR_SIMD_NO_UNROLL_LOOP_START
10415+ do {
10416+ float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels;
10417+ float const * hc = horizontal_coefficients;
10418+
10419+ stbir__4_coeff_start();
10420+ stbir__3_coeff_remnant(4);
10421+ stbir__store_output();
10422+ } while ( output < output_end );
10423+}
10424+
10425+static void STBIR_chans( stbir__horizontal_gather_,_channels_with_8_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width )
10426+{
10427+ float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels;
10428+ float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer;
10429+ STBIR_SIMD_NO_UNROLL_LOOP_START
10430+ do {
10431+ float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels;
10432+ float const * hc = horizontal_coefficients;
10433+ stbir__4_coeff_start();
10434+ stbir__4_coeff_continue_from_4(4);
10435+ stbir__store_output();
10436+ } while ( output < output_end );
10437+}
10438+
10439+static void STBIR_chans( stbir__horizontal_gather_,_channels_with_9_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width )
10440+{
10441+ float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels;
10442+ float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer;
10443+ STBIR_SIMD_NO_UNROLL_LOOP_START
10444+ do {
10445+ float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels;
10446+ float const * hc = horizontal_coefficients;
10447+ stbir__4_coeff_start();
10448+ stbir__4_coeff_continue_from_4(4);
10449+ stbir__1_coeff_remnant(8);
10450+ stbir__store_output();
10451+ } while ( output < output_end );
10452+}
10453+
10454+static void STBIR_chans( stbir__horizontal_gather_,_channels_with_10_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width )
10455+{
10456+ float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels;
10457+ float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer;
10458+ STBIR_SIMD_NO_UNROLL_LOOP_START
10459+ do {
10460+ float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels;
10461+ float const * hc = horizontal_coefficients;
10462+ stbir__4_coeff_start();
10463+ stbir__4_coeff_continue_from_4(4);
10464+ stbir__2_coeff_remnant(8);
10465+ stbir__store_output();
10466+ } while ( output < output_end );
10467+}
10468+
10469+static void STBIR_chans( stbir__horizontal_gather_,_channels_with_11_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width )
10470+{
10471+ float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels;
10472+ float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer;
10473+ stbir__3_coeff_setup();
10474+ STBIR_SIMD_NO_UNROLL_LOOP_START
10475+ do {
10476+ float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels;
10477+ float const * hc = horizontal_coefficients;
10478+ stbir__4_coeff_start();
10479+ stbir__4_coeff_continue_from_4(4);
10480+ stbir__3_coeff_remnant(8);
10481+ stbir__store_output();
10482+ } while ( output < output_end );
10483+}
10484+
10485+static void STBIR_chans( stbir__horizontal_gather_,_channels_with_12_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width )
10486+{
10487+ float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels;
10488+ float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer;
10489+ STBIR_SIMD_NO_UNROLL_LOOP_START
10490+ do {
10491+ float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels;
10492+ float const * hc = horizontal_coefficients;
10493+ stbir__4_coeff_start();
10494+ stbir__4_coeff_continue_from_4(4);
10495+ stbir__4_coeff_continue_from_4(8);
10496+ stbir__store_output();
10497+ } while ( output < output_end );
10498+}
10499+
10500+static void STBIR_chans( stbir__horizontal_gather_,_channels_with_n_coeffs_mod0 )( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width )
10501+{
10502+ float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels;
10503+ float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer;
10504+ STBIR_SIMD_NO_UNROLL_LOOP_START
10505+ do {
10506+ float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels;
10507+ int n = ( ( horizontal_contributors->n1 - horizontal_contributors->n0 + 1 ) - 4 + 3 ) >> 2;
10508+ float const * hc = horizontal_coefficients;
10509+
10510+ stbir__4_coeff_start();
10511+ STBIR_SIMD_NO_UNROLL_LOOP_START
10512+ do {
10513+ hc += 4;
10514+ decode += STBIR__horizontal_channels * 4;
10515+ stbir__4_coeff_continue_from_4( 0 );
10516+ --n;
10517+ } while ( n > 0 );
10518+ stbir__store_output();
10519+ } while ( output < output_end );
10520+}
10521+
10522+static void STBIR_chans( stbir__horizontal_gather_,_channels_with_n_coeffs_mod1 )( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width )
10523+{
10524+ float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels;
10525+ float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer;
10526+ STBIR_SIMD_NO_UNROLL_LOOP_START
10527+ do {
10528+ float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels;
10529+ int n = ( ( horizontal_contributors->n1 - horizontal_contributors->n0 + 1 ) - 5 + 3 ) >> 2;
10530+ float const * hc = horizontal_coefficients;
10531+
10532+ stbir__4_coeff_start();
10533+ STBIR_SIMD_NO_UNROLL_LOOP_START
10534+ do {
10535+ hc += 4;
10536+ decode += STBIR__horizontal_channels * 4;
10537+ stbir__4_coeff_continue_from_4( 0 );
10538+ --n;
10539+ } while ( n > 0 );
10540+ stbir__1_coeff_remnant( 4 );
10541+ stbir__store_output();
10542+ } while ( output < output_end );
10543+}
10544+
10545+static void STBIR_chans( stbir__horizontal_gather_,_channels_with_n_coeffs_mod2 )( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width )
10546+{
10547+ float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels;
10548+ float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer;
10549+ STBIR_SIMD_NO_UNROLL_LOOP_START
10550+ do {
10551+ float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels;
10552+ int n = ( ( horizontal_contributors->n1 - horizontal_contributors->n0 + 1 ) - 6 + 3 ) >> 2;
10553+ float const * hc = horizontal_coefficients;
10554+
10555+ stbir__4_coeff_start();
10556+ STBIR_SIMD_NO_UNROLL_LOOP_START
10557+ do {
10558+ hc += 4;
10559+ decode += STBIR__horizontal_channels * 4;
10560+ stbir__4_coeff_continue_from_4( 0 );
10561+ --n;
10562+ } while ( n > 0 );
10563+ stbir__2_coeff_remnant( 4 );
10564+
10565+ stbir__store_output();
10566+ } while ( output < output_end );
10567+}
10568+
10569+static void STBIR_chans( stbir__horizontal_gather_,_channels_with_n_coeffs_mod3 )( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width )
10570+{
10571+ float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels;
10572+ float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer;
10573+ stbir__3_coeff_setup();
10574+ STBIR_SIMD_NO_UNROLL_LOOP_START
10575+ do {
10576+ float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels;
10577+ int n = ( ( horizontal_contributors->n1 - horizontal_contributors->n0 + 1 ) - 7 + 3 ) >> 2;
10578+ float const * hc = horizontal_coefficients;
10579+
10580+ stbir__4_coeff_start();
10581+ STBIR_SIMD_NO_UNROLL_LOOP_START
10582+ do {
10583+ hc += 4;
10584+ decode += STBIR__horizontal_channels * 4;
10585+ stbir__4_coeff_continue_from_4( 0 );
10586+ --n;
10587+ } while ( n > 0 );
10588+ stbir__3_coeff_remnant( 4 );
10589+
10590+ stbir__store_output();
10591+ } while ( output < output_end );
10592+}
10593+
10594+static stbir__horizontal_gather_channels_func * STBIR_chans(stbir__horizontal_gather_,_channels_with_n_coeffs_funcs)[4]=
10595+{
10596+ STBIR_chans(stbir__horizontal_gather_,_channels_with_n_coeffs_mod0),
10597+ STBIR_chans(stbir__horizontal_gather_,_channels_with_n_coeffs_mod1),
10598+ STBIR_chans(stbir__horizontal_gather_,_channels_with_n_coeffs_mod2),
10599+ STBIR_chans(stbir__horizontal_gather_,_channels_with_n_coeffs_mod3),
10600+};
10601+
10602+static stbir__horizontal_gather_channels_func * STBIR_chans(stbir__horizontal_gather_,_channels_funcs)[12]=
10603+{
10604+ STBIR_chans(stbir__horizontal_gather_,_channels_with_1_coeff),
10605+ STBIR_chans(stbir__horizontal_gather_,_channels_with_2_coeffs),
10606+ STBIR_chans(stbir__horizontal_gather_,_channels_with_3_coeffs),
10607+ STBIR_chans(stbir__horizontal_gather_,_channels_with_4_coeffs),
10608+ STBIR_chans(stbir__horizontal_gather_,_channels_with_5_coeffs),
10609+ STBIR_chans(stbir__horizontal_gather_,_channels_with_6_coeffs),
10610+ STBIR_chans(stbir__horizontal_gather_,_channels_with_7_coeffs),
10611+ STBIR_chans(stbir__horizontal_gather_,_channels_with_8_coeffs),
10612+ STBIR_chans(stbir__horizontal_gather_,_channels_with_9_coeffs),
10613+ STBIR_chans(stbir__horizontal_gather_,_channels_with_10_coeffs),
10614+ STBIR_chans(stbir__horizontal_gather_,_channels_with_11_coeffs),
10615+ STBIR_chans(stbir__horizontal_gather_,_channels_with_12_coeffs),
10616+};
10617+
10618+#undef STBIR__horizontal_channels
10619+#undef STB_IMAGE_RESIZE_DO_HORIZONTALS
10620+#undef stbir__1_coeff_only
10621+#undef stbir__1_coeff_remnant
10622+#undef stbir__2_coeff_only
10623+#undef stbir__2_coeff_remnant
10624+#undef stbir__3_coeff_only
10625+#undef stbir__3_coeff_remnant
10626+#undef stbir__3_coeff_setup
10627+#undef stbir__4_coeff_start
10628+#undef stbir__4_coeff_continue_from_4
10629+#undef stbir__store_output
10630+#undef stbir__store_output_tiny
10631+#undef STBIR_chans
10632+
10633+#endif // HORIZONALS
10634+
10635+#undef STBIR_strs_join2
10636+#undef STBIR_strs_join1
10637+
10638+#endif // STB_IMAGE_RESIZE_DO_HORIZONTALS/VERTICALS/CODERS
10639+
10640+/*
10641+------------------------------------------------------------------------------
10642+This software is available under 2 licenses -- choose whichever you prefer.
10643+------------------------------------------------------------------------------
10644+ALTERNATIVE A - MIT License
10645+Copyright (c) 2017 Sean Barrett
10646+Permission is hereby granted, free of charge, to any person obtaining a copy of
10647+this software and associated documentation files (the "Software"), to deal in
10648+the Software without restriction, including without limitation the rights to
10649+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
10650+of the Software, and to permit persons to whom the Software is furnished to do
10651+so, subject to the following conditions:
10652+The above copyright notice and this permission notice shall be included in all
10653+copies or substantial portions of the Software.
10654+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
10655+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
10656+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
10657+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
10658+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
10659+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
10660+SOFTWARE.
10661+------------------------------------------------------------------------------
10662+ALTERNATIVE B - Public Domain (www.unlicense.org)
10663+This is free and unencumbered software released into the public domain.
10664+Anyone is free to copy, modify, publish, use, compile, sell, or distribute this
10665+software, either in source code form or as a compiled binary, for any purpose,
10666+commercial or non-commercial, and by any means.
10667+In jurisdictions that recognize copyright laws, the author or authors of this
10668+software dedicate any and all copyright interest in the software to the public
10669+domain. We make this dedication for the benefit of the public at large and to
10670+the detriment of our heirs and successors. We intend this dedication to be an
10671+overt act of relinquishment in perpetuity of all present and future rights to
10672+this software under copyright law.
10673+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
10674+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
10675+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
10676+AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
10677+ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
10678+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
10679+------------------------------------------------------------------------------
10680+*/
+51,
-0
1@@ -0,0 +1,51 @@
2+#ifndef STBI_ALLOC_H
3+#define STBI_ALLOC_H
4+
5+#include <sys/mman.h>
6+#include <unistd.h>
7+#include <string.h>
8+#include <stdint.h>
9+
10+#define STBI_MALLOC(sz) stbi_mmap_alloc(sz)
11+#define STBI_FREE(p) stbi_mmap_free(p)
12+#define STBI_REALLOC(p,sz) stbi_mmap_realloc(p,sz)
13+
14+static void *stbi_mmap_alloc(size_t sz)
15+{
16+ if (sz == 0) return NULL;
17+ long page = sysconf(_SC_PAGESIZE);
18+ size_t mapsz = ((sz + page) & ~(page - 1)) + page;
19+ void *p = mmap(NULL, mapsz, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
20+ if (p == MAP_FAILED) return NULL;
21+ *(size_t*)p = mapsz;
22+ return (char*)p + page;
23+}
24+
25+static void stbi_mmap_free(void *p)
26+{
27+ if (!p) return;
28+ long page = sysconf(_SC_PAGESIZE);
29+ void *base = (char*)p - page;
30+ size_t mapsz = *(size_t*)base;
31+ munmap(base, mapsz);
32+}
33+
34+static void *stbi_mmap_realloc(void *p, size_t newsz)
35+{
36+ if (!p) return stbi_mmap_alloc(newsz);
37+ if (newsz == 0) {
38+ stbi_mmap_free(p);
39+ return NULL;
40+ }
41+ long page = sysconf(_SC_PAGESIZE);
42+ void *base = (char*)p - page;
43+ size_t old_mapsz = *(size_t*)base;
44+ size_t oldsz = old_mapsz - page;
45+ void *newp = stbi_mmap_alloc(newsz);
46+ if (!newp) return p;
47+ memcpy(newp, p, oldsz < newsz ? oldsz : newsz);
48+ munmap(base, old_mapsz);
49+ return newp;
50+}
51+
52+#endif