The shell-command works if I use the function as below.

(defun my-dired-open-execute ()
  (interactive)
  (let* ((file-name (dired-get-file-for-visit))
         (file-ext (file-name-extension file-name))
         (case-fold-search t))
    (if (file-exists-p file-name)
        (cond
         ((and (string-match "^r$" file-ext) (file-remote-p file-name))
          (let ((default-directory (or (and (eq major-mode 'dired-mode)
                                            (dired-current-directory))
                                       default-directory)))
            (shell-command
             (format "nohup /bin/apps64/software/R/R CMD BATCH --no-restore
--no-save --slave \"%s\" &"
                     (file-name-nondirectory file-name)))
            ))
         (t (w32-shell-execute "open" file-name nil 1))))))