Blender  V3.3
gpu_shader_dependency.cc
Go to the documentation of this file.
1 /* SPDX-License-Identifier: GPL-2.0-or-later
2  * Copyright 2021 Blender Foundation. All rights reserved. */
3 
11 #include <algorithm>
12 #include <iomanip>
13 #include <iostream>
14 #include <sstream>
15 
16 #include "BLI_ghash.h"
17 #include "BLI_map.hh"
18 #include "BLI_string_ref.hh"
19 
20 #include "gpu_material_library.h"
23 
24 extern "C" {
25 #define SHADER_SOURCE(datatoc, filename, filepath) extern char datatoc[];
26 #include "glsl_draw_source_list.h"
27 #include "glsl_gpu_source_list.h"
28 #ifdef WITH_OCIO
29 # include "glsl_ocio_source_list.h"
30 #endif
31 #undef SHADER_SOURCE
32 }
33 
34 namespace blender::gpu {
35 
38 
39 struct GPUSource {
44  bool dependencies_init = false;
46  std::string processed_source;
47 
48  GPUSource(const char *path,
49  const char *file,
50  const char *datatoc,
52  : fullpath(path), filename(file), source(datatoc)
53  {
54  /* Scan for builtins. */
55  /* FIXME: This can trigger false positive caused by disabled #if blocks. */
56  /* TODO(fclem): Could be made faster by scanning once. */
57  if (source.find("gl_FragCoord", 0)) {
59  }
60  if (source.find("gl_FrontFacing", 0)) {
62  }
63  if (source.find("gl_GlobalInvocationID", 0)) {
65  }
66  if (source.find("gl_InstanceID", 0)) {
68  }
69  if (source.find("gl_LocalInvocationID", 0)) {
71  }
72  if (source.find("gl_LocalInvocationIndex", 0)) {
74  }
75  if (source.find("gl_NumWorkGroup", 0)) {
77  }
78  if (source.find("gl_PointCoord", 0)) {
80  }
81  if (source.find("gl_PointSize", 0)) {
83  }
84  if (source.find("gl_PrimitiveID", 0)) {
86  }
87  if (source.find("gl_VertexID", 0)) {
89  }
90  if (source.find("gl_WorkGroupID", 0)) {
92  }
93  if (source.find("gl_WorkGroupSize", 0)) {
95  }
96 
97  /* TODO(fclem): We could do that at compile time. */
98  /* Limit to shared header files to avoid the temptation to use C++ syntax in .glsl files. */
99  if (filename.endswith(".h") || filename.endswith(".hh")) {
100  enum_preprocess();
102  }
103  else {
104  check_no_quotes();
105  }
106 
107  if (is_from_material_library()) {
109  }
110  };
111 
113  {
114  return (input.rfind("/*", offset) > input.rfind("*/", offset)) ||
115  (input.rfind("//", offset) > input.rfind("\n", offset));
116  }
117 
118  template<bool check_whole_word = true, bool reversed = false, typename T>
119  static int64_t find_str(const StringRef &input, const T keyword, int64_t offset = 0)
120  {
121  while (true) {
122  if constexpr (reversed) {
123  offset = input.rfind(keyword, offset);
124  }
125  else {
126  offset = input.find(keyword, offset);
127  }
128  if (offset > 0) {
129  if constexpr (check_whole_word) {
130  /* Fix false positive if something has "enum" as suffix. */
131  char previous_char = input[offset - 1];
132  if (!(ELEM(previous_char, '\n', '\t', ' ', ':', '(', ','))) {
133  offset += (reversed) ? -1 : 1;
134  continue;
135  }
136  }
137  /* Fix case where the keyword is in a comment. */
138  if (is_in_comment(input, offset)) {
139  offset += (reversed) ? -1 : 1;
140  continue;
141  }
142  }
143  return offset;
144  }
145  }
146 
147 #define find_keyword find_str<true, false>
148 #define rfind_keyword find_str<true, true>
149 #define find_token find_str<false, false>
150 #define rfind_token find_str<false, true>
151 
152  void print_error(const StringRef &input, int64_t offset, const StringRef message)
153  {
154  StringRef sub = input.substr(0, offset);
155  int64_t line_number = std::count(sub.begin(), sub.end(), '\n') + 1;
156  int64_t line_end = input.find("\n", offset);
157  int64_t line_start = input.rfind("\n", offset) + 1;
158  int64_t char_number = offset - line_start + 1;
159 
160  /* TODO Use clog. */
161 
162  std::cout << fullpath << ":" << line_number << ":" << char_number;
163 
164  std::cout << " error: " << message << "\n";
165  std::cout << std::setw(5) << line_number << " | "
166  << input.substr(line_start, line_end - line_start) << "\n";
167  std::cout << " | ";
168  for (int64_t i = 0; i < char_number - 1; i++) {
169  std::cout << " ";
170  }
171  std::cout << "^\n";
172  }
173 
174 #define CHECK(test_value, str, ofs, msg) \
175  if ((test_value) == -1) { \
176  print_error(str, ofs, msg); \
177  continue; \
178  }
179 
186  {
187 #ifdef DEBUG
188  int64_t pos = -1;
189  do {
190  pos = source.find('"', pos + 1);
191  if (pos == -1) {
192  break;
193  }
194  if (!is_in_comment(source, pos)) {
195  print_error(source, pos, "Quote characters are forbidden in GLSL files");
196  }
197  } while (true);
198 #endif
199  }
200 
207  {
208  if (source.find_first_of('"') == -1) {
209  return;
210  }
211 
213  std::replace(processed_source.begin(), processed_source.end(), '"', ' ');
214 
215  source = processed_source.c_str();
216  }
217 
252  {
253  const StringRefNull input = source;
254  std::string output;
255  int64_t cursor = -1;
256  int64_t last_pos = 0;
257  const bool is_cpp = filename.endswith(".hh");
258 
259  while (true) {
260  cursor = find_keyword(input, "enum ", cursor + 1);
261  if (cursor == -1) {
262  break;
263  }
264  /* Skip matches like `typedef enum myEnum myType;` */
265  if (cursor >= 8 && input.substr(cursor - 8, 8) == "typedef ") {
266  continue;
267  }
268  /* Output anything between 2 enums blocks. */
269  output += input.substr(last_pos, cursor - last_pos);
270 
271  /* Extract enum type name. */
272  int64_t name_start = input.find(" ", cursor);
273 
274  int64_t values_start = find_token(input, '{', cursor);
275  CHECK(values_start, input, cursor, "Malformed enum class. Expected \'{\' after typename.");
276 
277  StringRef enum_name = input.substr(name_start, values_start - name_start);
278  if (is_cpp) {
279  int64_t name_end = find_token(enum_name, ":");
280  CHECK(name_end, input, name_start, "Expected \':\' after C++ enum name.");
281 
282  int64_t underlying_type = find_keyword(enum_name, "uint32_t", name_end);
283  CHECK(underlying_type, input, name_start, "C++ enums needs uint32_t underlying type.");
284 
285  enum_name = input.substr(name_start, name_end);
286  }
287 
288  output += "#define " + enum_name + " uint\n";
289 
290  /* Extract enum values. */
291  int64_t values_end = find_token(input, '}', values_start);
292  CHECK(values_end, input, cursor, "Malformed enum class. Expected \'}\' after values.");
293 
294  /* Skip opening brackets. */
295  values_start += 1;
296 
297  StringRef enum_values = input.substr(values_start, values_end - values_start);
298 
299  /* Really poor check. Could be done better. */
300  int64_t token = find_token(enum_values, '{');
301  int64_t not_found = (token == -1) ? 0 : -1;
302  CHECK(not_found, input, values_start + token, "Unexpected \'{\' token inside enum values.");
303 
304  /* Do not capture the comma after the last value (if present). */
305  int64_t last_equal = rfind_token(enum_values, '=', values_end);
306  int64_t last_comma = rfind_token(enum_values, ',', values_end);
307  if (last_comma > last_equal) {
308  enum_values = input.substr(values_start, last_comma);
309  }
310 
311  output += "const uint " + enum_values;
312 
313  int64_t semicolon_found = (input[values_end + 1] == ';') ? 0 : -1;
314  CHECK(semicolon_found, input, values_end + 1, "Expected \';\' after enum type declaration.");
315 
316  /* Skip the curly bracket but not the semicolon. */
317  cursor = last_pos = values_end + 1;
318  }
319  /* If nothing has been changed, do not allocate processed_source. */
320  if (last_pos == 0) {
321  return;
322  }
323 
324  if (last_pos != 0) {
325  output += input.substr(last_pos);
326  }
327 
329  source = processed_source.c_str();
330  };
331 
333  {
334  const StringRefNull input = source;
335 
336  const char whitespace_chars[] = " \r\n\t";
337 
338  auto function_parse = [&](const StringRef input,
339  int64_t &cursor,
340  StringRef &out_return_type,
341  StringRef &out_name,
342  StringRef &out_args) -> bool {
343  cursor = find_keyword(input, "void ", cursor + 1);
344  if (cursor == -1) {
345  return false;
346  }
347  int64_t arg_start = find_token(input, '(', cursor);
348  if (arg_start == -1) {
349  return false;
350  }
351  int64_t arg_end = find_token(input, ')', arg_start);
352  if (arg_end == -1) {
353  return false;
354  }
355  int64_t body_start = find_token(input, '{', arg_end);
356  int64_t next_semicolon = find_token(input, ';', arg_end);
357  if (body_start != -1 && next_semicolon != -1 && body_start > next_semicolon) {
358  /* Assert no prototypes but could also just skip them. */
359  BLI_assert_msg(false, "No prototypes allowed in node GLSL libraries.");
360  }
361  int64_t name_start = input.find_first_not_of(whitespace_chars, input.find(' ', cursor));
362  if (name_start == -1) {
363  return false;
364  }
365  int64_t name_end = input.find_last_not_of(whitespace_chars, arg_start);
366  if (name_end == -1) {
367  return false;
368  }
369  /* Only support void type for now. */
370  out_return_type = "void";
371  out_name = input.substr(name_start, name_end - name_start);
372  out_args = input.substr(arg_start + 1, arg_end - (arg_start + 1));
373  return true;
374  };
375 
376  auto keyword_parse = [&](const StringRef str, int64_t &cursor) -> StringRef {
377  int64_t keyword_start = str.find_first_not_of(whitespace_chars, cursor);
378  if (keyword_start == -1) {
379  /* No keyword found. */
380  return str.substr(0, 0);
381  }
382  int64_t keyword_end = str.find_first_of(whitespace_chars, keyword_start);
383  if (keyword_end == -1) {
384  /* Last keyword. */
385  keyword_end = str.size();
386  }
387  cursor = keyword_end + 1;
388  return str.substr(keyword_start, keyword_end - keyword_start);
389  };
390 
391  auto arg_parse = [&](const StringRef str,
392  int64_t &cursor,
393  StringRef &out_qualifier,
394  StringRef &out_type,
395  StringRef &out_name) -> bool {
396  int64_t arg_start = cursor + 1;
397  if (arg_start >= str.size()) {
398  return false;
399  }
400  cursor = find_token(str, ',', arg_start);
401  if (cursor == -1) {
402  /* Last argument. */
403  cursor = str.size();
404  }
405  const StringRef arg = str.substr(arg_start, cursor - arg_start);
406 
407  int64_t keyword_cursor = 0;
408  out_qualifier = keyword_parse(arg, keyword_cursor);
409  out_type = keyword_parse(arg, keyword_cursor);
410  out_name = keyword_parse(arg, keyword_cursor);
411  if (out_name.is_empty()) {
412  /* No qualifier case. */
413  out_name = out_type;
414  out_type = out_qualifier;
415  out_qualifier = arg.substr(0, 0);
416  }
417  return true;
418  };
419 
420  int64_t cursor = -1;
421  StringRef func_return_type, func_name, func_args;
422  while (function_parse(input, cursor, func_return_type, func_name, func_args)) {
423  GPUFunction *func = MEM_new<GPUFunction>(__func__);
424  func_name.copy(func->name, sizeof(func->name));
425  func->source = reinterpret_cast<void *>(this);
426 
427  bool insert = g_functions->add(func->name, func);
428 
429  /* NOTE: We allow overloading non void function, but only if the function comes from the
430  * same file. Otherwise the dependency system breaks. */
431  if (!insert) {
432  GPUSource *other_source = reinterpret_cast<GPUSource *>(
433  g_functions->lookup(func_name)->source);
434  if (other_source != this) {
436  source.find(func_name),
437  "Function redefinition or overload in two different files ...");
438  print_error(
439  input, other_source->source.find(func_name), "... previous definition was here");
440  }
441  else {
442  /* Non-void function overload. */
443  MEM_delete(func);
444  }
445  continue;
446  }
447 
448  if (func_return_type != "void") {
449  continue;
450  }
451 
452  func->totparam = 0;
453  int64_t args_cursor = -1;
454  StringRef arg_qualifier, arg_type, arg_name;
455  while (arg_parse(func_args, args_cursor, arg_qualifier, arg_type, arg_name)) {
456 
457  if (func->totparam >= ARRAY_SIZE(func->paramtype)) {
458  print_error(input, source.find(func_name), "Too much parameter in function");
459  break;
460  }
461 
462  auto parse_qualifier = [](StringRef qualifier) -> GPUFunctionQual {
463  if (qualifier == "out") {
464  return FUNCTION_QUAL_OUT;
465  }
466  if (qualifier == "inout") {
467  return FUNCTION_QUAL_INOUT;
468  }
469  return FUNCTION_QUAL_IN;
470  };
471 
472  auto parse_type = [](StringRef type) -> eGPUType {
473  if (type == "float") {
474  return GPU_FLOAT;
475  }
476  if (type == "vec2") {
477  return GPU_VEC2;
478  }
479  if (type == "vec3") {
480  return GPU_VEC3;
481  }
482  if (type == "vec4") {
483  return GPU_VEC4;
484  }
485  if (type == "mat3") {
486  return GPU_MAT3;
487  }
488  if (type == "mat4") {
489  return GPU_MAT4;
490  }
491  if (type == "sampler1DArray") {
492  return GPU_TEX1D_ARRAY;
493  }
494  if (type == "sampler2DArray") {
495  return GPU_TEX2D_ARRAY;
496  }
497  if (type == "sampler2D") {
498  return GPU_TEX2D;
499  }
500  if (type == "sampler3D") {
501  return GPU_TEX3D;
502  }
503  if (type == "Closure") {
504  return GPU_CLOSURE;
505  }
506  return GPU_NONE;
507  };
508 
509  func->paramqual[func->totparam] = parse_qualifier(arg_qualifier);
510  func->paramtype[func->totparam] = parse_type(arg_type);
511 
512  if (func->paramtype[func->totparam] == GPU_NONE) {
513  std::string err = "Unknown parameter type \"" + arg_type + "\"";
514  int64_t err_ofs = source.find(func_name);
515  err_ofs = find_keyword(source, arg_name, err_ofs);
516  err_ofs = rfind_keyword(source, arg_type, err_ofs);
517  print_error(input, err_ofs, err);
518  }
519 
520  func->totparam++;
521  }
522  }
523  }
524 
525 #undef find_keyword
526 #undef rfind_keyword
527 #undef find_token
528 #undef rfind_token
529 
530  /* Return 1 one error. */
533  {
534  if (this->dependencies_init) {
535  return 0;
536  }
537  this->dependencies_init = true;
538  int64_t pos = -1;
539 
540  while (true) {
541  GPUSource *dependency_source = nullptr;
542 
543  {
544  pos = source.find("pragma BLENDER_REQUIRE(", pos + 1);
545  if (pos == -1) {
546  return 0;
547  }
548  int64_t start = source.find('(', pos) + 1;
549  int64_t end = source.find(')', pos);
550  if (end == -1) {
551  print_error(source, start, "Malformed BLENDER_REQUIRE: Missing \")\" token");
552  return 1;
553  }
554  StringRef dependency_name = source.substr(start, end - start);
555  dependency_source = dict.lookup_default(dependency_name, nullptr);
556  if (dependency_source == nullptr) {
557  print_error(source, start, "Dependency not found");
558  return 1;
559  }
560  }
561  /* Recursive. */
562  int result = dependency_source->init_dependencies(dict, g_functions);
563  if (result != 0) {
564  return 1;
565  }
566 
567  for (auto *dep : dependency_source->dependencies) {
568  dependencies.append_non_duplicates(dep);
569  }
570  dependencies.append_non_duplicates(dependency_source);
571  }
572  return 0;
573  }
574 
575  /* Returns the final string with all includes done. */
577  {
578  for (auto *dep : dependencies) {
579  result.append(dep->source.c_str());
580  }
581  result.append(source.c_str());
582  }
583 
585  {
587  for (auto *dep : dependencies) {
588  out_builtins |= dep->builtins;
589  }
590  return out_builtins;
591  }
592 
594  {
595  return (filename.startswith("gpu_shader_material_") ||
596  filename.startswith("gpu_shader_common_")) &&
597  filename.endswith(".glsl");
598  }
599 };
600 
601 } // namespace blender::gpu
602 
603 using namespace blender::gpu;
604 
605 static GPUSourceDictionnary *g_sources = nullptr;
607 
609 {
612 
613 #define SHADER_SOURCE(datatoc, filename, filepath) \
614  g_sources->add_new(filename, new GPUSource(filepath, filename, datatoc, g_functions));
615 #include "glsl_draw_source_list.h"
616 #include "glsl_gpu_source_list.h"
617 #ifdef WITH_OCIO
618 # include "glsl_ocio_source_list.h"
619 #endif
620 #undef SHADER_SOURCE
621 
622  int errors = 0;
623  for (auto *value : g_sources->values()) {
624  errors += value->init_dependencies(*g_sources, *g_functions);
625  }
626  BLI_assert_msg(errors == 0, "Dependency errors detected: Aborting");
627  UNUSED_VARS_NDEBUG(errors);
628 }
629 
631 {
632  for (auto *value : g_sources->values()) {
633  delete value;
634  }
635  for (auto *value : g_functions->values()) {
636  MEM_delete(value);
637  }
638  delete g_sources;
639  delete g_functions;
640 }
641 
642 GPUFunction *gpu_material_library_use_function(GSet *used_libraries, const char *name)
643 {
644  GPUFunction *function = g_functions->lookup_default(name, nullptr);
645  BLI_assert_msg(function != nullptr, "Requested function not in the function library");
646  GPUSource *source = reinterpret_cast<GPUSource *>(function->source);
647  BLI_gset_add(used_libraries, const_cast<char *>(source->filename.c_str()));
648  return function;
649 }
650 
651 namespace blender::gpu::shader {
652 
654 {
655  if (shader_source_name.is_empty()) {
657  }
658  if (g_sources->contains(shader_source_name) == false) {
659  std::cout << "Error: Could not find \"" << shader_source_name
660  << "\" in the list of registered source.\n";
661  BLI_assert(0);
663  }
664  GPUSource *source = g_sources->lookup(shader_source_name);
665  return source->builtins_get();
666 }
667 
669  const StringRefNull shader_source_name)
670 {
672  GPUSource *src = g_sources->lookup_default(shader_source_name, nullptr);
673  if (src == nullptr) {
674  std::cout << "Error source not found : " << shader_source_name << std::endl;
675  }
676  src->build(result);
677  return result;
678 }
679 
681 {
682  GPUSource *src = g_sources->lookup_default(shader_source_name, nullptr);
683  if (src == nullptr) {
684  std::cout << "Error source not found : " << shader_source_name << std::endl;
685  }
686  return src->source;
687 }
688 
690  const StringRefNull source_string)
691 {
692  for (auto &source : g_sources->values()) {
693  if (source->source.c_str() == source_string.c_str()) {
694  return source->filename;
695  }
696  }
697  return "";
698 }
699 
700 } // namespace blender::gpu::shader
#define BLI_assert(a)
Definition: BLI_assert.h:46
#define BLI_assert_msg(a, msg)
Definition: BLI_assert.h:53
struct GSet GSet
Definition: BLI_ghash.h:340
bool BLI_gset_add(GSet *gs, void *key)
Definition: BLI_ghash.c:969
#define ARRAY_SIZE(arr)
#define UNUSED_VARS_NDEBUG(...)
#define ELEM(...)
_GL_VOID GLfloat value _GL_VOID_RET _GL_VOID const GLuint GLboolean *residences _GL_BOOL_RET _GL_VOID GLsizei GLfloat GLfloat GLfloat GLfloat const GLubyte *bitmap _GL_VOID_RET _GL_VOID GLenum type
eGPUType
Definition: GPU_material.h:45
@ GPU_VEC2
Definition: GPU_material.h:50
@ GPU_MAT4
Definition: GPU_material.h:54
@ GPU_TEX1D_ARRAY
Definition: GPU_material.h:58
@ GPU_TEX2D_ARRAY
Definition: GPU_material.h:60
@ GPU_TEX2D
Definition: GPU_material.h:59
@ GPU_VEC4
Definition: GPU_material.h:52
@ GPU_NONE
Definition: GPU_material.h:48
@ GPU_CLOSURE
Definition: GPU_material.h:64
@ GPU_VEC3
Definition: GPU_material.h:51
@ GPU_MAT3
Definition: GPU_material.h:53
@ GPU_TEX3D
Definition: GPU_material.h:61
@ GPU_FLOAT
Definition: GPU_material.h:49
bool add(const Key &key, const Value &value)
Definition: BLI_map.hh:250
Value lookup_default(const Key &key, const Value &default_value) const
Definition: BLI_map.hh:510
ValueIterator values() const
Definition: BLI_map.hh:840
const Value & lookup(const Key &key) const
Definition: BLI_map.hh:485
bool contains(const Key &key) const
Definition: BLI_map.hh:308
void copy(char *dst, int64_t dst_size) const
constexpr int64_t find(char c, int64_t pos=0) const
constexpr const char * begin() const
constexpr const char * end() const
constexpr bool is_empty() const
constexpr StringRef substr(int64_t start, int64_t size) const
constexpr bool startswith(StringRef prefix) const
constexpr bool endswith(StringRef suffix) const
constexpr int64_t find_first_of(StringRef chars, int64_t pos=0) const
constexpr const char * c_str() const
FILE * file
SyclQueue void void * src
#define str(s)
uint pos
GPUFunctionQual
@ FUNCTION_QUAL_IN
@ FUNCTION_QUAL_OUT
@ FUNCTION_QUAL_INOUT
#define rfind_token
#define find_token
#define rfind_keyword
void gpu_shader_dependency_init()
GPUFunction * gpu_material_library_use_function(GSet *used_libraries, const char *name)
void gpu_shader_dependency_exit()
static GPUFunctionDictionnary * g_functions
#define find_keyword
#define CHECK(test_value, str, ofs, msg)
static GPUSourceDictionnary * g_sources
int count
ccl_global KernelShaderEvalInput ccl_global float * output
ccl_gpu_kernel_postfix ccl_global float int int int int float bool int offset
ccl_global KernelShaderEvalInput * input
#define T
Insertion insert(const float3 &point_prev, const float3 &handle_prev, const float3 &handle_next, const float3 &point_next, float parameter)
Definition: curve_bezier.cc:61
StringRefNull gpu_shader_dependency_get_filename_from_source_string(const StringRefNull source_string)
BuiltinBits gpu_shader_dependency_get_builtins(const StringRefNull shader_source_name)
StringRefNull gpu_shader_dependency_get_source(const StringRefNull shader_source_name)
Vector< const char * > gpu_shader_dependency_get_resolved_source(const StringRefNull shader_source_name)
Map< StringRef, struct GPUSource * > GPUSourceDictionnary
Map< StringRef, struct GPUFunction * > GPUFunctionDictionnary
__int64 int64_t
Definition: stdint.h:89
eGPUType paramtype[MAX_PARAMETER]
char name[MAX_FUNCTION_NAME]
GPUFunctionQual paramqual[MAX_PARAMETER]
Vector< GPUSource * > dependencies
int init_dependencies(const GPUSourceDictionnary &dict, const GPUFunctionDictionnary &g_functions)
void material_functions_parse(GPUFunctionDictionnary *g_functions)
void build(Vector< const char * > &result) const
shader::BuiltinBits builtins_get() const
static int64_t find_str(const StringRef &input, const T keyword, int64_t offset=0)
shader::BuiltinBits builtins
GPUSource(const char *path, const char *file, const char *datatoc, GPUFunctionDictionnary *g_functions)
static bool is_in_comment(const StringRef &input, int64_t offset)
void print_error(const StringRef &input, int64_t offset, const StringRef message)
static FT_Error err