Parser: handle command substitution in lexer rules

Inside command substitution, many characters are allowed. For example,
MY_PV="$(replace_version_separator 1 '')" will reserve the single quotes
for command substitution. Now we support this by handling command
substitution in the lexer rules and call back to parser in walker
grammar.

6 files changed, 19 insertions, 19 deletions

diff --git a/bashast/gunit/array.gunit b/bashast/gunit/array.gunit
index ce0b344..0a176eb 100644
--- a/bashast/gunit/array.gunit
+++ b/bashast/gunit/array.gunit
@@ -20,9 +20,9 @@ gunit java_libbash;
 
 var_def:
 "asdf=(a b c d)"->(= asdf (ARRAY (STRING a) (STRING b) (STRING c) (STRING d)))
-"asdf=(`echo 6` b c d)"-> (= asdf (ARRAY (STRING (COMMAND_SUB (LIST (COMMAND (STRING echo) (STRING 6))))) (STRING b) (STRING c) (STRING d)))
+"asdf=(`echo 6` b c d)"-> (= asdf (ARRAY (STRING (COMMAND_SUB `echo 6`)) (STRING b) (STRING c) (STRING d)))
 "asdf=(${P} b c d)"->(= asdf (ARRAY (STRING (VAR_REF P)) (STRING b) (STRING c) (STRING d)))
-"asdf=($(echo a))" -> (= asdf (ARRAY (STRING (COMMAND_SUB (LIST (COMMAND (STRING echo) (STRING a)))))))
+"asdf=($(echo a))" -> (= asdf (ARRAY (STRING (COMMAND_SUB $(echo a)))))
 "asdf=(
 		--disable-dependency-tracking
 		${VAR}
diff --git a/bashast/gunit/command_sub.gunit b/bashast/gunit/command_sub.gunit
index e2d0021..49c5844 100644
--- a/bashast/gunit/command_sub.gunit
+++ b/bashast/gunit/command_sub.gunit
@@ -19,8 +19,8 @@
 gunit java_libbash;
 
 command_sub:
-"$(echo \"foo\")" -> (COMMAND_SUB (LIST (COMMAND (STRING echo) (STRING (DOUBLE_QUOTED_STRING foo)))))
-"$(ls |grep file)" -> (COMMAND_SUB (LIST (| (COMMAND (STRING ls)) (COMMAND (STRING grep) (STRING file)))))
-"$(CONTROL= command arg )" -> (COMMAND_SUB (LIST (COMMAND (STRING command) (STRING arg) (= CONTROL))))
-"`cat output.log |grep error|cut`" -> (COMMAND_SUB (LIST (| (| (COMMAND (STRING cat) (STRING output . log)) (COMMAND (STRING grep) (STRING error))) (COMMAND (STRING cut)))))
-"$(function foo() { echo 'hello'; }; foo)" -> (COMMAND_SUB (LIST (COMMAND (function (STRING foo) (CURRENT_SHELL (LIST (COMMAND (STRING echo) (STRING (SINGLE_QUOTED_STRING 'hello'))))))) (COMMAND (STRING foo))))
+"$(echo \"foo\")" -> (COMMAND_SUB $(echo "foo"))
+"$(ls |grep file)" -> (COMMAND_SUB $(ls |grep file))
+"$(CONTROL= command arg )" -> (COMMAND_SUB $(CONTROL= command arg ))
+"`cat output.log |grep error|cut`" -> (COMMAND_SUB `cat output.log |grep error|cut`)
+"$(function foo() { echo 'hello'; }; foo)" -> (COMMAND_SUB $(function foo() { echo 'hello'; }; foo))
diff --git a/bashast/gunit/compound.gunit b/bashast/gunit/compound.gunit
index 5e2a098..bf98197 100644
--- a/bashast/gunit/compound.gunit
+++ b/bashast/gunit/compound.gunit
@@ -54,7 +54,7 @@ subshell:
 case_expr:
 "case a in esac" -> (case (STRING a))
 "case `echo asdf` in
-esac" -> (case (STRING (COMMAND_SUB (LIST (COMMAND (STRING echo) (STRING asdf))))))
+esac" -> (case (STRING (COMMAND_SUB `echo asdf`)))
 
 "case `echo asdf` in
 gz)
@@ -65,7 +65,7 @@ echo three
 ;;
 *) echo woo
 ;;
-esac" -> (case (STRING (COMMAND_SUB (LIST (COMMAND (STRING echo) (STRING asdf))))) (CASE_PATTERN (STRING gz) CASE_COMMAND (LIST (COMMAND (STRING echo) (STRING yay)))) (CASE_PATTERN (STRING bzip) CASE_COMMAND (LIST (COMMAND (STRING echo) (STRING three)))) (CASE_PATTERN (STRING MATCH_ALL) CASE_COMMAND (LIST (COMMAND (STRING echo) (STRING woo)))))
+esac" -> (case (STRING (COMMAND_SUB `echo asdf`)) (CASE_PATTERN (STRING gz) CASE_COMMAND (LIST (COMMAND (STRING echo) (STRING yay)))) (CASE_PATTERN (STRING bzip) CASE_COMMAND (LIST (COMMAND (STRING echo) (STRING three)))) (CASE_PATTERN (STRING MATCH_ALL) CASE_COMMAND (LIST (COMMAND (STRING echo) (STRING woo)))))
 
 "case asdf in
 	gz)
@@ -83,8 +83,8 @@ esac" -> (case (STRING asdf) (CASE_PATTERN (STRING gz) CASE_COMMAND (LIST (COMMA
 for_expr:
 "for each in `ls |grep log`; do
 	echo \"file found\"
-done" -> (for each (STRING (COMMAND_SUB (LIST (| (COMMAND (STRING ls)) (COMMAND (STRING grep) (STRING log)))))) (LIST (COMMAND (STRING echo) (STRING (DOUBLE_QUOTED_STRING file   found)))))
-"for each in `ls |grep log`; do echo \"file found\"; done" -> (for each (STRING (COMMAND_SUB (LIST (| (COMMAND (STRING ls)) (COMMAND (STRING grep) (STRING log)))))) (LIST (COMMAND (STRING echo) (STRING (DOUBLE_QUOTED_STRING file   found)))))
+done" -> (for each (STRING (COMMAND_SUB `ls |grep log`)) (LIST (COMMAND (STRING echo) (STRING (DOUBLE_QUOTED_STRING file   found)))))
+"for each in `ls |grep log`; do echo \"file found\"; done" -> (for each (STRING (COMMAND_SUB `ls |grep log`)) (LIST (COMMAND (STRING echo) (STRING (DOUBLE_QUOTED_STRING file   found)))))
 "for i in 'foo' 'bar'; do echo $i; done" -> (for i (STRING (SINGLE_QUOTED_STRING 'foo')) (STRING (SINGLE_QUOTED_STRING 'bar')) (LIST (COMMAND (STRING echo) (STRING (VAR_REF i)))))
 "for i in foo$var bar; do echo $i; done" -> (for i (STRING foo (VAR_REF var)) (STRING bar) (LIST (COMMAND (STRING echo) (STRING (VAR_REF i)))))
 "for each in `ls |grep log`; do echo file done" FAIL
@@ -97,8 +97,8 @@ done" -> (for each (STRING (COMMAND_SUB (LIST (| (COMMAND (STRING ls)) (COMMAND
 sel_expr:
 "select each in `ls |grep log`; do
   echo \"file found\"
-	done" -> (select each (STRING (COMMAND_SUB (LIST (| (COMMAND (STRING ls)) (COMMAND (STRING grep) (STRING log)))))) (LIST (COMMAND (STRING echo) (STRING (DOUBLE_QUOTED_STRING file   found)))))
-"select each in `ls |grep log`; do echo \"file found\"; done" -> (select each (STRING (COMMAND_SUB (LIST (| (COMMAND (STRING ls)) (COMMAND (STRING grep) (STRING log)))))) (LIST (COMMAND (STRING echo) (STRING (DOUBLE_QUOTED_STRING file   found)))))
+	done" -> (select each (STRING (COMMAND_SUB `ls |grep log`)) (LIST (COMMAND (STRING echo) (STRING (DOUBLE_QUOTED_STRING file   found)))))
+"select each in `ls |grep log`; do echo \"file found\"; done" -> (select each (STRING (COMMAND_SUB `ls |grep log`)) (LIST (COMMAND (STRING echo) (STRING (DOUBLE_QUOTED_STRING file   found)))))
 "select each in `ls |grep log`; do echo file done" FAIL
 
 if_expr:
@@ -185,7 +185,7 @@ command:
 "case a in esac >/dev/null" -> (COMMAND (case (STRING a)) (REDIR > (STRING / dev / null)))
 "for i in foo$var bar; do echo $i; done >/dev/null" -> (COMMAND (for i (STRING foo (VAR_REF var)) (STRING bar) (LIST (COMMAND (STRING echo) (STRING (VAR_REF i))))) (REDIR > (STRING / dev / null)))
 "for ((5+3;;5+3)); do echo yay; done >/dev/null" -> (COMMAND (CFOR (FOR_INIT (+ 5 3)) (LIST (COMMAND (STRING echo) (STRING yay))) (FOR_MOD (+ 5 3))) (REDIR > (STRING / dev / null)))
-"select each in `ls |grep log`; do echo \"file found\"; done >/dev/null" -> (COMMAND (select each (STRING (COMMAND_SUB (LIST (| (COMMAND (STRING ls)) (COMMAND (STRING grep) (STRING log)))))) (LIST (COMMAND (STRING echo) (STRING (DOUBLE_QUOTED_STRING file   found))))) (REDIR > (STRING / dev / null)))
+"select each in `ls |grep log`; do echo \"file found\"; done >/dev/null" -> (COMMAND (select each (STRING (COMMAND_SUB `ls |grep log`)) (LIST (COMMAND (STRING echo) (STRING (DOUBLE_QUOTED_STRING file   found))))) (REDIR > (STRING / dev / null)))
 
 "if echo yay2;
 then
diff --git a/bashast/gunit/expansions.gunit b/bashast/gunit/expansions.gunit
index a36d4da..b10931d 100644
--- a/bashast/gunit/expansions.gunit
+++ b/bashast/gunit/expansions.gunit
@@ -26,5 +26,5 @@ clist:
 "[[ while = while ]] && echo true" -> (LIST (&& (COMMAND (COMPOUND_COND (KEYWORD_TEST (= (STRING while) (STRING while))))) (COMMAND (STRING echo) (STRING true))))
 "for each in `ls |grep output`; do
 echo $each
-done" -> (LIST (COMMAND (for each (STRING (COMMAND_SUB (LIST (| (COMMAND (STRING ls)) (COMMAND (STRING grep) (STRING output)))))) (LIST (COMMAND (STRING echo) (STRING (VAR_REF each)))))))
+done" -> (LIST (COMMAND (for each (STRING (COMMAND_SUB `ls |grep output`)) (LIST (COMMAND (STRING echo) (STRING (VAR_REF each)))))))
 "wc <(cat /usr/share/dict/linux.words)" -> (LIST (COMMAND (STRING wc) (PROCESS_SUBSTITUTION < (LIST (COMMAND (STRING cat) (STRING / usr / share / dict / linux . words))))))
diff --git a/bashast/gunit/fname.gunit b/bashast/gunit/fname.gunit
index eb13235..882afd1 100644
--- a/bashast/gunit/fname.gunit
+++ b/bashast/gunit/fname.gunit
@@ -25,8 +25,8 @@ fname:
 "\"\"" -> (STRING DOUBLE_QUOTED_STRING)
 "\"For more info on this patchset, and how to report problems, see:\"" -> (STRING (DOUBLE_QUOTED_STRING For   more   info   on   this   patchset ,   and   how   to   report   problems ,   see :))
 "\"asdf;!\"" -> (STRING (DOUBLE_QUOTED_STRING asdf ; !))
-"\"$(whoami)\"" -> (STRING (DOUBLE_QUOTED_STRING (COMMAND_SUB (LIST (COMMAND (STRING whoami))))))
-"\"`whoami`\"" -> (STRING (DOUBLE_QUOTED_STRING (COMMAND_SUB (LIST (COMMAND (STRING whoami))))))
+"\"$(whoami)\"" -> (STRING (DOUBLE_QUOTED_STRING (COMMAND_SUB $(whoami))))
+"\"`whoami`\"" -> (STRING (DOUBLE_QUOTED_STRING (COMMAND_SUB `whoami`)))
 "'`whoami`'" -> (STRING (SINGLE_QUOTED_STRING '`whoami`'))
 "'$(whoami)'" -> (STRING (SINGLE_QUOTED_STRING '$(whoami)'))
 "\"${P}\"" -> (STRING (DOUBLE_QUOTED_STRING (VAR_REF P)))
@@ -72,4 +72,4 @@ fname:
 "\"abc#$/\"" -> (STRING (DOUBLE_QUOTED_STRING abc # $ /))
 
 dqstr:
-"\"\\\\\"\$\`\"" -> (DOUBLE_QUOTED_STRING \ " $ `)
+"\"\\\\\"\$\`\"" -> (DOUBLE_QUOTED_STRING \ " \$ \`)
diff --git a/bashast/gunit/simp_prog.gunit b/bashast/gunit/simp_prog.gunit
index 64aa001..d36c5f6 100644
--- a/bashast/gunit/simp_prog.gunit
+++ b/bashast/gunit/simp_prog.gunit
@@ -25,7 +25,7 @@ echo \"I found a file\"
 done
 
 echo \"finding complete\"
-"-> (LIST (COMMAND (STRING echo) (STRING (DOUBLE_QUOTED_STRING finding   file))) (COMMAND (for each (STRING (COMMAND_SUB (LIST (| (COMMAND (STRING ls)) (COMMAND (STRING grep) (STRING output)))))) (LIST (COMMAND (STRING echo) (STRING (DOUBLE_QUOTED_STRING I   found   a   file)))))) (COMMAND (STRING echo) (STRING (DOUBLE_QUOTED_STRING finding   complete))))
+"-> (LIST (COMMAND (STRING echo) (STRING (DOUBLE_QUOTED_STRING finding   file))) (COMMAND (for each (STRING (COMMAND_SUB `ls | grep output`)) (LIST (COMMAND (STRING echo) (STRING (DOUBLE_QUOTED_STRING I   found   a   file)))))) (COMMAND (STRING echo) (STRING (DOUBLE_QUOTED_STRING finding   complete))))
 
 "function quit {
   exit