Answer by egreg for Check for group as argument (expandable)

This ignores spaces (but doesn't deal with the edge case):

\documentclass{article}\usepackage{xparse}\ExplSyntaxOn\NewDocumentCommand{\MyIfGroup}{mmm} {  \regex_match:nnTF { \A\s*\cB\{.*\cE\}\s*\Z } { #1 } { #2 } { #3 } }\ExplSyntaxOff\begin{document}\MyIfGroup{{}}{true}{false}   $\to$ true\MyIfGroup{ {}}{true}{false}  $\to$ true\MyIfGroup{{} }{true}{false}  $\to$ true\MyIfGroup{ {} }{true}{false} $\to$ true\MyIfGroup{ }{true}{false}    $\to$ false\MyIfGroup{a}{true}{false}    $\to$ false\MyIfGroup{{}a}{true}{false}  $\to$ false\end{document}

If you want to deal with the edge case and make it return false:

\documentclass{article}\usepackage{xparse}\ExplSyntaxOn\NewDocumentCommand{\MyIfGroup}{mmm} {  \regex_match:nnTF { \A\s*\cB\{.*\cE\}\s*\Z } { #1 }   {    \int_compare:nNnTF { \tl_count:n { #1 } } > { 1 } { #3 } { #2 }   }   { #3 } }\ExplSyntaxOff\begin{document}\MyIfGroup{{}}{true}{false}   $\to$ true\MyIfGroup{ {}}{true}{false}  $\to$ true\MyIfGroup{{} }{true}{false}  $\to$ true\MyIfGroup{ {} }{true}{false} $\to$ true\MyIfGroup{ }{true}{false}    $\to$ false\MyIfGroup{a}{true}{false}    $\to$ false\MyIfGroup{{}a}{true}{false}  $\to$ false\MyIfGroup{{}{}}{true}{false} $\to$ false\end{document}