1use super::ParsingError;
2use super::syntax_definition::*;
3use super::scope::*;
4
5#[cfg(feature = "metadata")]
6use super::metadata::{LoadMetadata, Metadata, RawMetadataEntry};
7
8#[cfg(feature = "yaml-load")]
9use super::super::LoadingError;
10
11use std::collections::{HashMap, HashSet, BTreeSet};
12use std::path::Path;
13use std::io::{self, BufRead, BufReader};
14use std::fs::File;
15use std::mem;
16
17use super::regex::Regex;
18use crate::parsing::syntax_definition::ContextId;
19use once_cell::sync::OnceCell;
20use serde_derive::{Deserialize, Serialize};
21
22#[derive(Debug, Serialize, Deserialize)]
32pub struct SyntaxSet {
33 syntaxes: Vec<SyntaxReference>,
34 path_syntaxes: Vec<(String, usize)>,
36
37 #[serde(skip_serializing, skip_deserializing, default = "OnceCell::new")]
38 first_line_cache: OnceCell<FirstLineCache>,
39 #[cfg(feature = "metadata")]
44 #[serde(skip, default)]
45 pub(crate) metadata: Metadata,
46}
47
48#[derive(Clone, Debug, Serialize, Deserialize)]
52pub struct SyntaxReference {
53 pub name: String,
54 pub file_extensions: Vec<String>,
55 pub scope: Scope,
56 pub first_line_match: Option<String>,
57 pub hidden: bool,
58 #[serde(serialize_with = "ordered_map")]
59 pub variables: HashMap<String, String>,
60 #[serde(skip)]
61 pub(crate) lazy_contexts: OnceCell<LazyContexts>,
62 pub(crate) serialized_lazy_contexts: Vec<u8>,
63}
64
65#[derive(Clone, Debug, Serialize, Deserialize)]
67pub(crate) struct LazyContexts {
68 #[serde(serialize_with = "ordered_map")]
69 pub(crate) context_ids: HashMap<String, ContextId>,
70 pub(crate) contexts: Vec<Context>,
71}
72
73#[derive(Clone, Default)]
83pub struct SyntaxSetBuilder {
84 syntaxes: Vec<SyntaxDefinition>,
85 path_syntaxes: Vec<(String, usize)>,
86 #[cfg(feature = "metadata")]
87 raw_metadata: LoadMetadata,
88
89 #[cfg(feature = "metadata")]
93 existing_metadata: Option<Metadata>,
94}
95
96#[cfg(feature = "yaml-load")]
97fn load_syntax_file(p: &Path,
98 lines_include_newline: bool)
99 -> Result<SyntaxDefinition, LoadingError> {
100 let s = std::fs::read_to_string(p)?;
101
102 SyntaxDefinition::load_from_str(
103 &s,
104 lines_include_newline,
105 p.file_stem().and_then(|x| x.to_str()),
106 )
107 .map_err(|e| LoadingError::ParseSyntax(e, format!("{}", p.display())))
108}
109
110impl Clone for SyntaxSet {
111 fn clone(&self) -> SyntaxSet {
112 SyntaxSet {
113 syntaxes: self.syntaxes.clone(),
114 path_syntaxes: self.path_syntaxes.clone(),
115 first_line_cache: OnceCell::new(),
117 #[cfg(feature = "metadata")]
118 metadata: self.metadata.clone(),
119 }
120 }
121}
122
123impl Default for SyntaxSet {
124 fn default() -> Self {
125 SyntaxSet {
126 syntaxes: Vec::new(),
127 path_syntaxes: Vec::new(),
128 first_line_cache: OnceCell::new(),
129 #[cfg(feature = "metadata")]
130 metadata: Metadata::default(),
131 }
132 }
133}
134
135impl SyntaxSet {
136 pub fn new() -> SyntaxSet {
137 SyntaxSet::default()
138 }
139
140 #[cfg(feature = "yaml-load")]
150 pub fn load_from_folder<P: AsRef<Path>>(folder: P) -> Result<SyntaxSet, LoadingError> {
151 let mut builder = SyntaxSetBuilder::new();
152 builder.add_from_folder(folder, false)?;
153 Ok(builder.build())
154 }
155
156 pub fn syntaxes(&self) -> &[SyntaxReference] {
158 &self.syntaxes[..]
159 }
160
161 #[cfg(feature = "metadata")]
162 pub fn set_metadata(&mut self, metadata: Metadata) {
163 self.metadata = metadata;
164 }
165
166 #[cfg(feature = "metadata")]
168 pub fn metadata(&self) -> &Metadata {
169 &self.metadata
170 }
171
172 pub fn find_syntax_by_scope(&self, scope: Scope) -> Option<&SyntaxReference> {
178 self.syntaxes.iter().rev().find(|&s| s.scope == scope)
179 }
180
181 pub fn find_syntax_by_name<'a>(&'a self, name: &str) -> Option<&'a SyntaxReference> {
182 self.syntaxes.iter().rev().find(|&s| name == s.name)
183 }
184
185 pub fn find_syntax_by_extension<'a>(&'a self, extension: &str) -> Option<&'a SyntaxReference> {
186 self.syntaxes.iter().rev().find(|&s| s.file_extensions.iter().any(|e| e.eq_ignore_ascii_case(extension)))
187 }
188
189 pub fn find_syntax_by_token<'a>(&'a self, s: &str) -> Option<&'a SyntaxReference> {
194 {
195 let ext_res = self.find_syntax_by_extension(s);
196 if ext_res.is_some() {
197 return ext_res;
198 }
199 }
200 self.syntaxes.iter().rev().find(|&syntax| syntax.name.eq_ignore_ascii_case(s))
201 }
202
203 pub fn find_syntax_by_first_line<'a>(&'a self, s: &str) -> Option<&'a SyntaxReference> {
208 let cache = self.first_line_cache();
209 for &(ref reg, i) in cache.regexes.iter().rev() {
210 if reg.search(s, 0, s.len(), None) {
211 return Some(&self.syntaxes[i]);
212 }
213 }
214 None
215 }
216
217 pub fn find_syntax_by_path<'a>(&'a self, path: &str) -> Option<&'a SyntaxReference> {
225 let mut slash_path = "/".to_string();
226 slash_path.push_str(path);
227 self.path_syntaxes.iter().rev().find(|t| t.0.ends_with(&slash_path) || t.0 == path).map(|&(_,i)| &self.syntaxes[i])
228 }
229
230 pub fn find_syntax_for_file<P: AsRef<Path>>(&self,
249 path_obj: P)
250 -> io::Result<Option<&SyntaxReference>> {
251 let path: &Path = path_obj.as_ref();
252 let file_name = path.file_name().and_then(|n| n.to_str()).unwrap_or("");
253 let extension = path.extension().and_then(|x| x.to_str()).unwrap_or("");
254 let ext_syntax = self.find_syntax_by_extension(file_name).or_else(
255 || self.find_syntax_by_extension(extension));
256 let line_syntax = if ext_syntax.is_none() {
257 let mut line = String::new();
258 let f = File::open(path)?;
259 let mut line_reader = BufReader::new(&f);
260 line_reader.read_line(&mut line)?;
261 self.find_syntax_by_first_line(&line)
262 } else {
263 None
264 };
265 let syntax = ext_syntax.or(line_syntax);
266 Ok(syntax)
267 }
268
269 pub fn find_syntax_plain_text(&self) -> &SyntaxReference {
289 self.find_syntax_by_name("Plain Text")
290 .expect("All syntax sets ought to have a plain text syntax")
291 }
292
293 pub fn into_builder(self) -> SyntaxSetBuilder {
299 #[cfg(feature = "metadata")]
300 let SyntaxSet { syntaxes, path_syntaxes, metadata, .. } = self;
301 #[cfg(not(feature = "metadata"))]
302 let SyntaxSet { syntaxes, path_syntaxes, .. } = self;
303
304 let mut context_map = HashMap::new();
305 for (syntax_index, syntax) in syntaxes.iter().enumerate() {
306 for (context_index, context) in syntax.contexts().iter().enumerate() {
307 context_map.insert(ContextId { syntax_index, context_index }, context.clone());
308 }
309 }
310
311 let mut builder_syntaxes = Vec::with_capacity(syntaxes.len());
312
313 for syntax in syntaxes {
314 let SyntaxReference {
315 name,
316 file_extensions,
317 scope,
318 first_line_match,
319 hidden,
320 variables,
321 serialized_lazy_contexts,
322 ..
323 } = syntax;
324
325 let lazy_contexts = LazyContexts::deserialize(&serialized_lazy_contexts[..]);
326 let mut builder_contexts = HashMap::with_capacity(lazy_contexts.context_ids.len());
327 for (name, context_id) in lazy_contexts.context_ids {
328 if let Some(context) = context_map.remove(&context_id) {
329 builder_contexts.insert(name, context);
330 }
331 }
332
333 let syntax_definition = SyntaxDefinition {
334 name,
335 file_extensions,
336 scope,
337 first_line_match,
338 hidden,
339 variables,
340 contexts: builder_contexts,
341 };
342 builder_syntaxes.push(syntax_definition);
343 }
344
345 SyntaxSetBuilder {
346 syntaxes: builder_syntaxes,
347 path_syntaxes,
348 #[cfg(feature = "metadata")]
349 existing_metadata: Some(metadata),
350 #[cfg(feature = "metadata")]
351 raw_metadata: LoadMetadata::default(),
352 }
353 }
354
355 #[inline(always)]
356 pub(crate) fn get_context(&self, context_id: &ContextId) -> Result<&Context, ParsingError> {
357 let syntax = &self
358 .syntaxes
359 .get(context_id.syntax_index)
360 .ok_or(ParsingError::MissingContext(*context_id))?;
361 syntax
362 .contexts()
363 .get(context_id.context_index)
364 .ok_or(ParsingError::MissingContext(*context_id))
365 }
366
367 fn first_line_cache(&self) -> &FirstLineCache {
368 self.first_line_cache
369 .get_or_init(|| FirstLineCache::new(self.syntaxes()))
370 }
371
372 pub fn find_unlinked_contexts(&self) -> BTreeSet<String> {
373 let SyntaxSet { syntaxes, .. } = self;
374
375 let mut unlinked_contexts = BTreeSet::new();
376
377 for syntax in syntaxes {
378 let SyntaxReference {
379 name,
380 scope,
381 ..
382 } = syntax;
383
384 for context in syntax.contexts() {
385 Self::find_unlinked_contexts_in_context(name, scope, context, &mut unlinked_contexts);
386 }
387 }
388 unlinked_contexts
389 }
390
391 fn find_unlinked_contexts_in_context(
392 name: &str,
393 scope: &Scope,
394 context: &Context,
395 unlinked_contexts: &mut BTreeSet<String>,
396 ) {
397 for pattern in context.patterns.iter() {
398 let maybe_refs_to_check = match pattern {
399 Pattern::Match(match_pat) => match &match_pat.operation {
400 MatchOperation::Push(context_refs) => Some(context_refs),
401 MatchOperation::Set(context_refs) => Some(context_refs),
402 _ => None,
403 },
404 _ => None,
405 };
406 for context_ref in maybe_refs_to_check.into_iter().flatten() {
407 match context_ref {
408 ContextReference::Direct(_) => {}
409 _ => {
410 unlinked_contexts.insert(format!(
411 "Syntax '{}' with scope '{}' has unresolved context reference {:?}",
412 name, scope, &context_ref
413 ));
414 }
415 }
416 }
417 }
418 }
419}
420
421impl SyntaxReference {
422 pub(crate) fn context_ids(&self) -> &HashMap<String, ContextId> {
423 &self.lazy_contexts().context_ids
424 }
425
426 fn contexts(&self) -> &[Context] {
427 &self.lazy_contexts().contexts
428 }
429
430 fn lazy_contexts(&self) -> &LazyContexts {
431 self.lazy_contexts
432 .get_or_init(|| LazyContexts::deserialize(&self.serialized_lazy_contexts[..]))
433 }
434}
435
436impl LazyContexts {
437 fn deserialize(data: &[u8]) -> LazyContexts {
438 crate::dumps::from_reader(data).expect("data is not corrupt or out of sync with the code")
439 }
440}
441
442impl SyntaxSetBuilder {
443 pub fn new() -> SyntaxSetBuilder {
444 SyntaxSetBuilder::default()
445 }
446
447 pub fn add(&mut self, syntax: SyntaxDefinition) {
449 self.syntaxes.push(syntax);
450 }
451
452 pub fn syntaxes(&self) -> &[SyntaxDefinition] {
454 &self.syntaxes[..]
455 }
456
457 #[cfg(feature = "yaml-load")]
462 pub fn add_plain_text_syntax(&mut self) {
463 let s = "---\nname: Plain Text\nfile_extensions: [txt]\nscope: text.plain\ncontexts: \
464 {main: []}";
465 let syn = SyntaxDefinition::load_from_str(s, false, None).unwrap();
466 self.syntaxes.push(syn);
467 }
468
469 #[cfg(feature = "yaml-load")]
483 pub fn add_from_folder<P: AsRef<Path>>(
484 &mut self,
485 folder: P,
486 lines_include_newline: bool
487 ) -> Result<(), LoadingError> {
488 for entry in crate::utils::walk_dir(folder).sort_by(|a, b| a.file_name().cmp(b.file_name())) {
489 let entry = entry.map_err(LoadingError::WalkDir)?;
490 if entry.path().extension().map_or(false, |e| e == "sublime-syntax") {
491 let syntax = load_syntax_file(entry.path(), lines_include_newline)?;
492 if let Some(path_str) = entry.path().to_str() {
493 let path = Path::new(path_str);
496 let path_parts: Vec<_> = path.iter().map(|c| c.to_str().unwrap()).collect();
497 self.path_syntaxes.push((path_parts.join("/").to_string(), self.syntaxes.len()));
498 }
499 self.syntaxes.push(syntax);
500 }
501
502 #[cfg(feature = "metadata")]
503 {
504 if entry.path().extension() == Some("tmPreferences".as_ref()) {
505 match RawMetadataEntry::load(entry.path()) {
506 Ok(meta) => self.raw_metadata.add_raw(meta),
507 Err(_err) => (),
508 }
509 }
510 }
511 }
512
513 Ok(())
514 }
515
516 pub fn build(self) -> SyntaxSet {
538
539 #[cfg(not(feature = "metadata"))]
540 let SyntaxSetBuilder { syntaxes: syntax_definitions, path_syntaxes } = self;
541 #[cfg(feature = "metadata")]
542 let SyntaxSetBuilder {
543 syntaxes: syntax_definitions,
544 path_syntaxes,
545 raw_metadata,
546 existing_metadata,
547 } = self;
548
549 let mut syntaxes = Vec::with_capacity(syntax_definitions.len());
550 let mut all_context_ids = Vec::new();
551 let mut all_contexts = vec![Vec::new(); syntax_definitions.len()];
552
553 for (syntax_index, syntax_definition) in syntax_definitions.into_iter().enumerate() {
554 let SyntaxDefinition {
555 name,
556 file_extensions,
557 scope,
558 first_line_match,
559 hidden,
560 variables,
561 contexts,
562 } = syntax_definition;
563
564 let mut context_ids = HashMap::new();
565
566 let mut contexts: Vec<(String, Context)> = contexts.into_iter().collect();
567 contexts.sort_unstable_by(|(name_a, _), (name_b, _)| name_a.cmp(name_b));
572 for (name, context) in contexts {
573 let context_index = all_contexts[syntax_index].len();
574 context_ids.insert(name, ContextId { syntax_index, context_index });
575 all_contexts[syntax_index].push(context);
576 }
577
578 let syntax = SyntaxReference {
579 name,
580 file_extensions,
581 scope,
582 first_line_match,
583 hidden,
584 variables,
585 lazy_contexts: OnceCell::new(),
586 serialized_lazy_contexts: Vec::new(), };
588 syntaxes.push(syntax);
589 all_context_ids.push(context_ids);
590 }
591
592 let mut found_more_backref_includes = true;
593 for (syntax_index, _syntax) in syntaxes.iter().enumerate() {
594 let mut no_prototype = HashSet::new();
595 let prototype = all_context_ids[syntax_index].get("prototype");
596 if let Some(prototype_id) = prototype {
597 Self::recursively_mark_no_prototype(prototype_id, &all_context_ids[syntax_index], &all_contexts, &mut no_prototype);
599 }
600
601 for context_id in all_context_ids[syntax_index].values() {
602 let context = &mut all_contexts[context_id.syntax_index][context_id.context_index];
603 if let Some(prototype_id) = prototype {
604 if context.meta_include_prototype && !no_prototype.contains(context_id) {
605 context.prototype = Some(*prototype_id);
606 }
607 }
608 Self::link_context(context, syntax_index, &all_context_ids, &syntaxes);
609
610 if context.uses_backrefs {
611 found_more_backref_includes = true;
612 }
613 }
614 }
615
616 while found_more_backref_includes {
624 found_more_backref_includes = false;
625 for syntax_index in 0..syntaxes.len() {
628 for context_index in 0..all_contexts[syntax_index].len() {
629 let context = &all_contexts[syntax_index][context_index];
630 if !context.uses_backrefs && context.patterns.iter().any(|pattern| {
631 matches!(pattern, Pattern::Include(ContextReference::Direct(id)) if all_contexts[id.syntax_index][id.context_index].uses_backrefs)
632 }) {
633 let context = &mut all_contexts[syntax_index][context_index];
634 context.uses_backrefs = true;
635 found_more_backref_includes = true;
637 }
638 }
639 }
640 }
641
642 #[cfg(feature = "metadata")]
643 let metadata = match existing_metadata {
644 Some(existing) => existing.merged_with_raw(raw_metadata),
645 None => raw_metadata.into(),
646 };
647
648 for syntax in &mut syntaxes {
653 let lazy_contexts = LazyContexts {
654 context_ids: all_context_ids.remove(0),
655 contexts: all_contexts.remove(0),
656 };
657
658 syntax.serialized_lazy_contexts = crate::dumps::dump_binary(&lazy_contexts);
659 };
660
661 SyntaxSet {
662 syntaxes,
663 path_syntaxes,
664 first_line_cache: OnceCell::new(),
665 #[cfg(feature = "metadata")]
666 metadata,
667 }
668 }
669
670 fn recursively_mark_no_prototype(
673 context_id: &ContextId,
674 syntax_context_ids: &HashMap<String, ContextId>,
675 all_contexts: &[Vec<Context>],
676 no_prototype: &mut HashSet<ContextId>,
677 ) {
678 let first_time = no_prototype.insert(*context_id);
679 if !first_time {
680 return;
681 }
682
683 for pattern in &all_contexts[context_id.syntax_index][context_id.context_index].patterns {
684 match *pattern {
685 Pattern::Match(ref match_pat) => {
688 let maybe_context_refs = match match_pat.operation {
689 MatchOperation::Push(ref context_refs) |
690 MatchOperation::Set(ref context_refs) => Some(context_refs),
691 MatchOperation::Pop | MatchOperation::None => None,
692 };
693 if let Some(context_refs) = maybe_context_refs {
694 for context_ref in context_refs.iter() {
695 match context_ref {
696 ContextReference::Inline(ref s) | ContextReference::Named(ref s) => {
697 if let Some(i) = syntax_context_ids.get(s) {
698 Self::recursively_mark_no_prototype(i, syntax_context_ids, all_contexts, no_prototype);
699 }
700 },
701 ContextReference::Direct(ref id) => {
702 Self::recursively_mark_no_prototype(id, syntax_context_ids, all_contexts, no_prototype);
703 },
704 _ => (),
705 }
706 }
707 }
708 }
709 Pattern::Include(ref reference) => {
710 match reference {
711 ContextReference::Named(ref s) => {
712 if let Some(id) = syntax_context_ids.get(s) {
713 Self::recursively_mark_no_prototype(id, syntax_context_ids, all_contexts, no_prototype);
714 }
715 },
716 ContextReference::Direct(ref id) => {
717 Self::recursively_mark_no_prototype(id, syntax_context_ids, all_contexts, no_prototype);
718 },
719 _ => (),
720 }
721 }
722 }
723 }
724 }
725
726 fn link_context(
727 context: &mut Context,
728 syntax_index: usize,
729 all_context_ids: &[HashMap<String, ContextId>],
730 syntaxes: &[SyntaxReference],
731 ) {
732 for pattern in &mut context.patterns {
733 match *pattern {
734 Pattern::Match(ref mut match_pat) => Self::link_match_pat(match_pat, syntax_index, all_context_ids, syntaxes),
735 Pattern::Include(ref mut context_ref) => Self::link_ref(context_ref, syntax_index, all_context_ids, syntaxes),
736 }
737 }
738 }
739
740 fn link_ref(
741 context_ref: &mut ContextReference,
742 syntax_index: usize,
743 all_context_ids: &[HashMap<String, ContextId>],
744 syntaxes: &[SyntaxReference],
745 ) {
746 use super::syntax_definition::ContextReference::*;
748 let linked_context_id = match *context_ref {
749 Named(ref s) | Inline(ref s) => {
750 if s == "$top_level_main" {
754 all_context_ids[syntax_index].get("main")
755 } else {
756 all_context_ids[syntax_index].get(s)
757 }
758 }
759 ByScope { scope, ref sub_context, with_escape } => {
760 Self::with_plain_text_fallback(all_context_ids, syntaxes, with_escape, Self::find_id(sub_context, all_context_ids, syntaxes, |index_and_syntax| {
761 index_and_syntax.1.scope == scope
762 }))
763 }
764 File { ref name, ref sub_context, with_escape } => {
765 Self::with_plain_text_fallback(all_context_ids, syntaxes, with_escape, Self::find_id(sub_context, all_context_ids, syntaxes, |index_and_syntax| {
766 &index_and_syntax.1.name == name
767 }))
768 }
769 Direct(_) => None,
770 };
771 if let Some(context_id) = linked_context_id {
772 let mut new_ref = Direct(*context_id);
773 mem::swap(context_ref, &mut new_ref);
774 }
775 }
776
777 fn with_plain_text_fallback<'a>(
778 all_context_ids: &'a [HashMap<String, ContextId>],
779 syntaxes: &'a [SyntaxReference],
780 with_escape: bool,
781 context_id: Option<&'a ContextId>,
782 ) -> Option<&'a ContextId> {
783 context_id.or_else(|| {
784 if with_escape {
785 Self::find_id(
792 &None,
793 all_context_ids,
794 syntaxes,
795 |index_and_syntax| index_and_syntax.1.name == "Plain Text",
796 )
797 } else {
798 None
799 }
800 })
801 }
802
803 fn find_id<'a>(
804 sub_context: &Option<String>,
805 all_context_ids: &'a [HashMap<String, ContextId>],
806 syntaxes: &'a [SyntaxReference],
807 predicate: impl FnMut(&(usize, &SyntaxReference)) -> bool,
808 ) -> Option<&'a ContextId> {
809 let context_name = sub_context.as_ref().map_or("main", |x| &**x);
810 syntaxes
811 .iter()
812 .enumerate()
813 .rev()
814 .find(predicate)
815 .and_then(|index_and_syntax| all_context_ids[index_and_syntax.0].get(context_name))
816 }
817
818 fn link_match_pat(
819 match_pat: &mut MatchPattern,
820 syntax_index: usize,
821 all_context_ids: &[HashMap<String, ContextId>],
822 syntaxes: &[SyntaxReference],
823 ) {
824 let maybe_context_refs = match match_pat.operation {
825 MatchOperation::Push(ref mut context_refs) |
826 MatchOperation::Set(ref mut context_refs) => Some(context_refs),
827 MatchOperation::Pop | MatchOperation::None => None,
828 };
829 if let Some(context_refs) = maybe_context_refs {
830 for context_ref in context_refs.iter_mut() {
831 Self::link_ref(context_ref, syntax_index, all_context_ids, syntaxes);
832 }
833 }
834 if let Some(ref mut context_ref) = match_pat.with_prototype {
835 Self::link_ref(context_ref, syntax_index, all_context_ids, syntaxes);
836 }
837 }
838}
839
840#[derive(Debug)]
841struct FirstLineCache {
842 regexes: Vec<(Regex, usize)>,
844}
845
846impl FirstLineCache {
847 fn new(syntaxes: &[SyntaxReference]) -> FirstLineCache {
848 let mut regexes = Vec::new();
849 for (i, syntax) in syntaxes.iter().enumerate() {
850 if let Some(ref reg_str) = syntax.first_line_match {
851 let reg = Regex::new(reg_str.into());
852 regexes.push((reg, i));
853 }
854 }
855 FirstLineCache {
856 regexes,
857 }
858 }
859}
860
861
862#[cfg(feature = "yaml-load")]
863#[cfg(test)]
864mod tests {
865 use super::*;
866 use crate::parsing::{ParseState, Scope, syntax_definition};
867 use std::collections::HashMap;
868
869 #[test]
870 fn can_load() {
871 let mut builder = SyntaxSetBuilder::new();
872 builder.add_from_folder("testdata/Packages", false).unwrap();
873
874 let cmake_dummy_syntax = SyntaxDefinition {
875 name: "CMake".to_string(),
876 file_extensions: vec!["CMakeLists.txt".to_string(), "cmake".to_string()],
877 scope: Scope::new("source.cmake").unwrap(),
878 first_line_match: None,
879 hidden: false,
880 variables: HashMap::new(),
881 contexts: HashMap::new(),
882 };
883
884 builder.add(cmake_dummy_syntax);
885 builder.add_plain_text_syntax();
886
887 let ps = builder.build();
888
889 assert_eq!(&ps.find_syntax_by_first_line("#!/usr/bin/env node").unwrap().name,
890 "JavaScript");
891 let rails_scope = Scope::new("source.ruby.rails").unwrap();
892 let syntax = ps.find_syntax_by_name("Ruby on Rails").unwrap();
893 ps.find_syntax_plain_text();
894 assert_eq!(&ps.find_syntax_by_extension("rake").unwrap().name, "Ruby");
895 assert_eq!(&ps.find_syntax_by_extension("RAKE").unwrap().name, "Ruby");
896 assert_eq!(&ps.find_syntax_by_token("ruby").unwrap().name, "Ruby");
897 assert_eq!(&ps.find_syntax_by_first_line("lol -*- Mode: C -*- such line").unwrap().name,
898 "C");
899 assert_eq!(&ps.find_syntax_for_file("testdata/parser.rs").unwrap().unwrap().name,
900 "Rust");
901 assert_eq!(&ps.find_syntax_for_file("testdata/test_first_line.test")
902 .expect("Error finding syntax for file")
903 .expect("No syntax found for file")
904 .name,
905 "Ruby");
906 assert_eq!(&ps.find_syntax_for_file(".bashrc").unwrap().unwrap().name,
907 "Bourne Again Shell (bash)");
908 assert_eq!(&ps.find_syntax_for_file("CMakeLists.txt").unwrap().unwrap().name,
909 "CMake");
910 assert_eq!(&ps.find_syntax_for_file("test.cmake").unwrap().unwrap().name,
911 "CMake");
912 assert_eq!(&ps.find_syntax_for_file("Rakefile").unwrap().unwrap().name, "Ruby");
913 assert!(&ps.find_syntax_by_first_line("derp derp hi lol").is_none());
914 assert_eq!(&ps.find_syntax_by_path("Packages/Rust/Rust.sublime-syntax").unwrap().name,
915 "Rust");
916 assert_eq!(syntax.scope, rails_scope);
918 let main_context = ps.get_context(&syntax.context_ids()["main"]).expect("#[cfg(test)]");
920 let count = syntax_definition::context_iter(&ps, main_context).count();
921 assert_eq!(count, 109);
922 }
923
924 #[test]
925 fn can_clone() {
926 let cloned_syntax_set = {
927 let mut builder = SyntaxSetBuilder::new();
928 builder.add(syntax_a());
929 builder.add(syntax_b());
930
931 let syntax_set_original = builder.build();
932 #[allow(clippy::redundant_clone)] syntax_set_original.clone()
934 };
936
937 let syntax = cloned_syntax_set.find_syntax_by_extension("a").unwrap();
938 let mut parse_state = ParseState::new(syntax);
939 let ops = parse_state.parse_line("a go_b b", &cloned_syntax_set).expect("#[cfg(test)]");
940 let expected = (7, ScopeStackOp::Push(Scope::new("b").unwrap()));
941 assert_ops_contain(&ops, &expected);
942 }
943
944 #[test]
945 fn can_list_added_syntaxes() {
946 let mut builder = SyntaxSetBuilder::new();
947 builder.add(syntax_a());
948 builder.add(syntax_b());
949 let syntaxes = builder.syntaxes();
950
951 assert_eq!(syntaxes.len(), 2);
952 assert_eq!(syntaxes[0].name, "A");
953 assert_eq!(syntaxes[1].name, "B");
954 }
955
956 #[test]
957 fn can_add_more_syntaxes_with_builder() {
958 let syntax_set_original = {
959 let mut builder = SyntaxSetBuilder::new();
960 builder.add(syntax_a());
961 builder.add(syntax_b());
962 builder.build()
963 };
964
965 let mut builder = syntax_set_original.into_builder();
966
967 let syntax_c = SyntaxDefinition::load_from_str(r#"
968 name: C
969 scope: source.c
970 file_extensions: [c]
971 contexts:
972 main:
973 - match: 'c'
974 scope: c
975 - match: 'go_a'
976 push: scope:source.a#main
977 "#, true, None).unwrap();
978
979 builder.add(syntax_c);
980
981 let syntax_set = builder.build();
982
983 let syntax = syntax_set.find_syntax_by_extension("c").unwrap();
984 let mut parse_state = ParseState::new(syntax);
985 let ops = parse_state.parse_line("c go_a a go_b b", &syntax_set).expect("#[cfg(test)]");
986 let expected = (14, ScopeStackOp::Push(Scope::new("b").unwrap()));
987 assert_ops_contain(&ops, &expected);
988 }
989
990 #[test]
991 fn falls_back_to_plain_text_when_embedded_scope_is_missing() {
992 test_plain_text_fallback(r#"
993 name: Z
994 scope: source.z
995 file_extensions: [z]
996 contexts:
997 main:
998 - match: 'z'
999 scope: z
1000 - match: 'go_x'
1001 embed: scope:does.not.exist
1002 escape: 'leave_x'
1003 "#);
1004 }
1005
1006 #[test]
1007 fn falls_back_to_plain_text_when_embedded_file_is_missing() {
1008 test_plain_text_fallback(r#"
1009 name: Z
1010 scope: source.z
1011 file_extensions: [z]
1012 contexts:
1013 main:
1014 - match: 'z'
1015 scope: z
1016 - match: 'go_x'
1017 embed: DoesNotExist.sublime-syntax
1018 escape: 'leave_x'
1019 "#);
1020 }
1021
1022 fn test_plain_text_fallback(syntax_definition: &str) {
1023 let syntax =
1024 SyntaxDefinition::load_from_str(syntax_definition, true, None).unwrap();
1025
1026 let mut builder = SyntaxSetBuilder::new();
1027 builder.add_plain_text_syntax();
1028 builder.add(syntax);
1029 let syntax_set = builder.build();
1030
1031 let syntax = syntax_set.find_syntax_by_extension("z").unwrap();
1032 let mut parse_state = ParseState::new(syntax);
1033 let ops = parse_state.parse_line("z go_x x leave_x z", &syntax_set).unwrap();
1034 let expected_ops = vec![
1035 (0, ScopeStackOp::Push(Scope::new("source.z").unwrap())),
1036 (0, ScopeStackOp::Push(Scope::new("z").unwrap())),
1037 (1, ScopeStackOp::Pop(1)),
1038 (6, ScopeStackOp::Push(Scope::new("text.plain").unwrap())),
1039 (9, ScopeStackOp::Pop(1)),
1040 (17, ScopeStackOp::Push(Scope::new("z").unwrap())),
1041 (18, ScopeStackOp::Pop(1)),
1042 ];
1043 assert_eq!(ops, expected_ops);
1044 }
1045
1046 #[test]
1047 fn can_find_unlinked_contexts() {
1048 let syntax_set = {
1049 let mut builder = SyntaxSetBuilder::new();
1050 builder.add(syntax_a());
1051 builder.add(syntax_b());
1052 builder.build()
1053 };
1054
1055 let unlinked_contexts = syntax_set.find_unlinked_contexts();
1056 assert_eq!(unlinked_contexts.len(), 0);
1057
1058 let syntax_set = {
1059 let mut builder = SyntaxSetBuilder::new();
1060 builder.add(syntax_a());
1061 builder.build()
1062 };
1063
1064 let unlinked_contexts : Vec<String> = syntax_set.find_unlinked_contexts().into_iter().collect();
1065 assert_eq!(unlinked_contexts.len(), 1);
1066 assert_eq!(unlinked_contexts[0], "Syntax 'A' with scope 'source.a' has unresolved context reference ByScope { scope: <source.b>, sub_context: Some(\"main\"), with_escape: false }");
1067 }
1068
1069 #[test]
1070 fn can_use_in_multiple_threads() {
1071 use rayon::prelude::*;
1072
1073 let syntax_set = {
1074 let mut builder = SyntaxSetBuilder::new();
1075 builder.add(syntax_a());
1076 builder.add(syntax_b());
1077 builder.build()
1078 };
1079
1080 let lines = vec![
1081 "a a a",
1082 "a go_b b",
1083 "go_b b",
1084 "go_b b b",
1085 ];
1086
1087 let results: Vec<Vec<(usize, ScopeStackOp)>> = lines
1088 .par_iter()
1089 .map(|line| {
1090 let syntax = syntax_set.find_syntax_by_extension("a").unwrap();
1091 let mut parse_state = ParseState::new(syntax);
1092 parse_state.parse_line(line, &syntax_set).expect("#[cfg(test)]")
1093 })
1094 .collect();
1095
1096 assert_ops_contain(&results[0], &(4, ScopeStackOp::Push(Scope::new("a").unwrap())));
1097 assert_ops_contain(&results[1], &(7, ScopeStackOp::Push(Scope::new("b").unwrap())));
1098 assert_ops_contain(&results[2], &(5, ScopeStackOp::Push(Scope::new("b").unwrap())));
1099 assert_ops_contain(&results[3], &(8, ScopeStackOp::Push(Scope::new("b").unwrap())));
1100 }
1101
1102 #[test]
1103 fn is_sync() {
1104 check_sync::<SyntaxSet>();
1105 }
1106
1107 #[test]
1108 fn is_send() {
1109 check_send::<SyntaxSet>();
1110 }
1111
1112 #[test]
1113 fn can_override_syntaxes() {
1114 let syntax_set = {
1115 let mut builder = SyntaxSetBuilder::new();
1116 builder.add(syntax_a());
1117 builder.add(syntax_b());
1118
1119 let syntax_a2 = SyntaxDefinition::load_from_str(r#"
1120 name: A improved
1121 scope: source.a
1122 file_extensions: [a]
1123 first_line_match: syntax\s+a
1124 contexts:
1125 main:
1126 - match: a
1127 scope: a2
1128 - match: go_b
1129 push: scope:source.b#main
1130 "#, true, None).unwrap();
1131
1132 builder.add(syntax_a2);
1133
1134 let syntax_c = SyntaxDefinition::load_from_str(r#"
1135 name: C
1136 scope: source.c
1137 file_extensions: [c]
1138 first_line_match: syntax\s+.*
1139 contexts:
1140 main:
1141 - match: c
1142 scope: c
1143 - match: go_a
1144 push: scope:source.a#main
1145 "#, true, None).unwrap();
1146
1147 builder.add(syntax_c);
1148
1149 builder.build()
1150 };
1151
1152 let mut syntax = syntax_set.find_syntax_by_extension("a").unwrap();
1153 assert_eq!(syntax.name, "A improved");
1154 syntax = syntax_set.find_syntax_by_scope(Scope::new("source.a").unwrap()).unwrap();
1155 assert_eq!(syntax.name, "A improved");
1156 syntax = syntax_set.find_syntax_by_first_line("syntax a").unwrap();
1157 assert_eq!(syntax.name, "C");
1158
1159 let mut parse_state = ParseState::new(syntax);
1160 let ops = parse_state.parse_line("c go_a a", &syntax_set).expect("msg");
1161 let expected = (7, ScopeStackOp::Push(Scope::new("a2").unwrap()));
1162 assert_ops_contain(&ops, &expected);
1163 }
1164
1165 #[test]
1166 fn can_parse_issue219() {
1167 let syntax_set = SyntaxSet::load_defaults_newlines().into_builder().build();
1172 let syntax = syntax_set.find_syntax_by_extension("yaml").unwrap();
1173
1174 let mut parse_state = ParseState::new(syntax);
1175 let ops = parse_state.parse_line("# test\n", &syntax_set).expect("#[cfg(test)]");
1176 let expected = (0, ScopeStackOp::Push(Scope::new("comment.line.number-sign.yaml").unwrap()));
1177 assert_ops_contain(&ops, &expected);
1178 }
1179
1180 #[test]
1181 fn no_prototype_for_contexts_included_from_prototype() {
1182 let mut builder = SyntaxSetBuilder::new();
1183 let syntax = SyntaxDefinition::load_from_str(r#"
1184 name: Test Prototype
1185 scope: source.test
1186 file_extensions: [test]
1187 contexts:
1188 prototype:
1189 - include: included_from_prototype
1190 main:
1191 - match: main
1192 - match: other
1193 push: other
1194 other:
1195 - match: o
1196 included_from_prototype:
1197 - match: p
1198 scope: p
1199 "#, true, None).unwrap();
1200 builder.add(syntax);
1201 let ss = builder.build();
1202
1203 assert_prototype_only_on(&["main", "other"], &ss, &ss.syntaxes()[0]);
1206
1207 let rebuilt = ss.into_builder().build();
1211 assert_prototype_only_on(&["main", "other"], &rebuilt, &rebuilt.syntaxes()[0]);
1212 }
1213
1214 #[test]
1215 fn no_prototype_for_contexts_inline_in_prototype() {
1216 let mut builder = SyntaxSetBuilder::new();
1217 let syntax = SyntaxDefinition::load_from_str(r#"
1218 name: Test Prototype
1219 scope: source.test
1220 file_extensions: [test]
1221 contexts:
1222 prototype:
1223 - match: p
1224 push:
1225 - match: p2
1226 main:
1227 - match: main
1228 "#, true, None).unwrap();
1229 builder.add(syntax);
1230 let ss = builder.build();
1231
1232 assert_prototype_only_on(&["main"], &ss, &ss.syntaxes()[0]);
1233
1234 let rebuilt = ss.into_builder().build();
1235 assert_prototype_only_on(&["main"], &rebuilt, &rebuilt.syntaxes()[0]);
1236 }
1237
1238 fn assert_ops_contain(
1239 ops: &[(usize, ScopeStackOp)],
1240 expected: &(usize, ScopeStackOp)
1241 ) {
1242 assert!(ops.contains(expected),
1243 "expected operations to contain {:?}: {:?}", expected, ops);
1244 }
1245
1246 fn assert_prototype_only_on(expected: &[&str], syntax_set: &SyntaxSet, syntax: &SyntaxReference) {
1247 for (name, id) in syntax.context_ids() {
1248 if name == "__main" || name == "__start" {
1249 continue;
1251 }
1252 let context = syntax_set.get_context(id).expect("#[cfg(test)]");
1253 if expected.contains(&name.as_str()) {
1254 assert!(context.prototype.is_some(), "Expected context {} to have prototype", name);
1255 } else {
1256 assert!(context.prototype.is_none(), "Expected context {} to not have prototype", name);
1257 }
1258 }
1259 }
1260
1261 fn check_send<T: Send>() {}
1262
1263 fn check_sync<T: Sync>() {}
1264
1265 fn syntax_a() -> SyntaxDefinition {
1266 SyntaxDefinition::load_from_str(
1267 r#"
1268 name: A
1269 scope: source.a
1270 file_extensions: [a]
1271 contexts:
1272 main:
1273 - match: 'a'
1274 scope: a
1275 - match: 'go_b'
1276 push: scope:source.b#main
1277 "#,
1278 true,
1279 None,
1280 ).unwrap()
1281 }
1282
1283 fn syntax_b() -> SyntaxDefinition {
1284 SyntaxDefinition::load_from_str(
1285 r#"
1286 name: B
1287 scope: source.b
1288 file_extensions: [b]
1289 contexts:
1290 main:
1291 - match: 'b'
1292 scope: b
1293 "#,
1294 true,
1295 None,
1296 ).unwrap()
1297 }
1298}