does elasticsearch have capability to form es cluster with only one es docker service running multiple replicas of it in docker-swarm

yes it is possible to achieve it.
just add endpoint_mode as dnsrr like below in docker-compose.yaml file.
for deploying one instance/container across all docker-swarm nodes use "mode: global" in docker-compose.yaml file.
For deploying all ES cluster instance/containers in one machine, use "mode: replicated" and specify replicas to deploy as say "replicas: 3" in docker-compose.yaml file.

With usage of "endpoint_mode: dnsrr" we won't able to expose service ports.
For this purpose we should use an external load balancer like nginx or haproxy.
Like below nginx docker-compose.yaml file

version: '3.5'
services:
    
    elasticsearch:
       image: elasticsearch:5
       deploy:
#          mode: replicated
#          replicas: 3
          mode: global
          endpoint_mode: dnsrr
          restart_policy:
             condition: on-failure
             delay: 5s
             max_attempts: 3
             window: 120s
#       placement:
#           constraints: [node.role == manager]
       environment:
         - ES_JAVA_OPTS=${ELASTICSEARCH_MEM} 
       volumes:
         - esdata:/usr/share/elasticsearch/data
       configs:
         - source: elasticsearch.yml
           target: /usr/share/elasticsearch/config/elasticsearch.yml
         
#     ports:
#       - 9200:9200
#       - 9300:9300
    nginx-lb:
       image: "nginx:1.15.4"
       deploy:
           restart_policy:
              condition: on-failure
             delay: 5s
             max_attempts: 3
             window: 120s
           placement:
                constraints: [node.role == manager]
      volumes:
          - ./nginx.conf:/etc/nginx/nginx.conf
      ports:
          - 9200:9200
          - 9300:9300
  
volumes:
  esdata:
configs:
  elasticsearch.yml:
    external: true


#elasticsearch.yaml file
transport.host: 0.0.0.0
http.host: 0.0.0.0
network.host: 0.0.0.0
cluster.name: elasticsearch
discovery.zen.ping.unicast.hosts: elasticsearch  
discovery.zen.minimum_master_nodes: 1 
node.max_local_storage_nodes: 20

#nginx.conf file

events {}
stream {
    upstream stream_backend {
        server elasticsearch:9300;

    }
    
    
    server {
        listen        9300;
        proxy_pass    stream_backend;
        proxy_timeout 3s;
        proxy_connect_timeout 1s;
    }
    

}

http {

  upstream elasticsearchPort9200 {
      server elasticsearch:9200;


  }

  server {
      listen 9200;
      server_name _;

      location / {
        proxy_pass http://elasticsearchPort9200;
      }

  }

}