Clean up Shader Preprocessor

* Moved preprocessor to Shader and ShaderInclude
* Clean up RenderingServer side
* Preprocessor is separate from parser now, but it emits tokens with include location hints.
* Improved ShaderEditor validation code
* Added include file code completion
* Added notification for all files affected by a broken include.

scene/resources/shader.cpp

@@ -33,6 +33,7 @@
 #include "core/io/file_access.h"
 #include "scene/scene_string_names.h"
 #include "servers/rendering/shader_language.h"
+#include "servers/rendering/shader_preprocessor.h"
 #include "servers/rendering_server.h"
 #include "texture.h"
 
@@ -47,21 +48,17 @@ void Shader::_dependency_changed() {
 	emit_changed();
 }
 
-void Shader::set_code(const String &p_code) {
-	HashSet<Ref<ShaderInclude>> new_include_dependencies;
+void Shader::set_path(const String &p_path, bool p_take_over) {
+	Resource::set_path(p_path, p_take_over);
+	RS::get_singleton()->shader_set_path_hint(shader, p_path);
+}
 
+void Shader::set_code(const String &p_code) {
 	for (Ref<ShaderInclude> E : include_dependencies) {
 		E->disconnect(SNAME("changed"), callable_mp(this, &Shader::_dependency_changed));
 	}
 
-	String type = ShaderLanguage::get_shader_type_and_dependencies(p_code, &new_include_dependencies);
-
-	// This ensures previous include resources are not freed and then re-loaded during parse (which would make compiling slower)
-	include_dependencies = new_include_dependencies;
-
-	for (Ref<ShaderInclude> E : include_dependencies) {
-		E->connect(SNAME("changed"), callable_mp(this, &Shader::_dependency_changed));
-	}
+	String type = ShaderLanguage::get_shader_type(p_code);
 
 	if (type == "canvas_item") {
 		mode = MODE_CANVAS_ITEM;
@@ -75,7 +72,27 @@ void Shader::set_code(const String &p_code) {
 		mode = MODE_SPATIAL;
 	}
 
-	RenderingServer::get_singleton()->shader_set_code(shader, p_code);
+	code = p_code;
+	String pp_code = p_code;
+
+	HashSet<Ref<ShaderInclude>> new_include_dependencies;
+
+	{
+		// Preprocessor must run here and not in the server because:
+		// 1) Need to keep track of include dependencies at resource level
+		// 2) Server does not do interaction with Resource filetypes, this is a scene level feature.
+		ShaderPreprocessor preprocessor;
+		preprocessor.preprocess(p_code, pp_code, nullptr, nullptr, &new_include_dependencies);
+	}
+
+	// This ensures previous include resources are not freed and then re-loaded during parse (which would make compiling slower)
+	include_dependencies = new_include_dependencies;
+
+	for (Ref<ShaderInclude> E : include_dependencies) {
+		E->connect(SNAME("changed"), callable_mp(this, &Shader::_dependency_changed));
+	}
+
+	RenderingServer::get_singleton()->shader_set_code(shader, pp_code);
 	params_cache_dirty = true;
 
 	emit_changed();
@@ -83,7 +100,7 @@ void Shader::set_code(const String &p_code) {
 
 String Shader::get_code() const {
 	_update_shader();
-	return RenderingServer::get_singleton()->shader_get_code(shader);
+	return code;
 }
 
 void Shader::get_param_list(List<PropertyInfo> *p_params) const {

scene/resources/shader.h

@@ -55,6 +55,7 @@ private:
 	RID shader;
 	Mode mode = MODE_SPATIAL;
 	HashSet<Ref<ShaderInclude>> include_dependencies;
+	String code;
 
 	// hack the name of performance
 	// shaders keep a list of ShaderMaterial -> RenderingServer name translations, to make
@@ -72,6 +73,8 @@ public:
 	//void set_mode(Mode p_mode);
 	virtual Mode get_mode() const;
 
+	virtual void set_path(const String &p_path, bool p_take_over = false) override;
+
 	void set_code(const String &p_code);
 	String get_code() const;
 

scene/resources/shader_include.cpp

@@ -30,6 +30,7 @@
 
 #include "shader_include.h"
 #include "servers/rendering/shader_language.h"
+#include "servers/rendering/shader_preprocessor.h"
 
 void ShaderInclude::_dependency_changed() {
 	emit_changed();
@@ -43,7 +44,11 @@ void ShaderInclude::set_code(const String &p_code) {
 		E->disconnect(SNAME("changed"), callable_mp(this, &ShaderInclude::_dependency_changed));
 	}
 
-	ShaderLanguage::get_shader_dependencies(p_code, &new_dependencies);
+	{
+		String pp_code;
+		ShaderPreprocessor preprocessor;
+		preprocessor.preprocess(p_code, pp_code, nullptr, nullptr, &new_dependencies);
+	}
 
 	// This ensures previous include resources are not freed and then re-loaded during parse (which would make compiling slower)
 	dependencies = new_dependencies;