'Pyspark Session giving file not found when initializing spark session
I am trying to start a spark session and it keeps giving me a file not found error. I have searched online and found that I am basically using what everyone is saying, however I have not been able to find a solution.
I verified the spark to be the following:
Name: pyspark
Version: 3.0.1
Summary: Apache Spark Python API
Home-page: https://github.com/apache/spark/tree/master/python
Author: Spark Developers
Author-email: [email protected]
License: http://www.apache.org/licenses/LICENSE-2.0
Location: c:\users\alexr\anaconda3\lib\site-packages
Requires: py4j
I use this code:
import pyspark
from pyspark.sql import SparkSession
spark = SparkSession.builder.appName("Prac").getOrCreate()
And I get the following error:
---------------------------------------------------------------------------
FileNotFoundError Traceback (most recent call last)
<ipython-input-3-5ea8472534b6> in <module>
2 from pyspark.sql import SparkSession
3
----> 4 spark = SparkSession.builder.appName("Prac").getOrCreate()
~\Anaconda3\lib\site-packages\pyspark\sql\session.py in getOrCreate(self)
184 sparkConf.set(key, value)
185 # This SparkContext may be an existing one.
--> 186 sc = SparkContext.getOrCreate(sparkConf)
187 # Do not update `SparkConf` for existing `SparkContext`, as it's shared
188 # by all sessions.
~\Anaconda3\lib\site-packages\pyspark\context.py in getOrCreate(cls, conf)
374 with SparkContext._lock:
375 if SparkContext._active_spark_context is None:
--> 376 SparkContext(conf=conf or SparkConf())
377 return SparkContext._active_spark_context
378
~\Anaconda3\lib\site-packages\pyspark\context.py in __init__(self, master, appName, sparkHome, pyFiles, environment, batchSize, serializer, conf, gateway, jsc, profiler_cls)
131 " is not allowed as it is a security risk.")
132
--> 133 SparkContext._ensure_initialized(self, gateway=gateway, conf=conf)
134 try:
135 self._do_init(master, appName, sparkHome, pyFiles, environment, batchSize, serializer,
~\Anaconda3\lib\site-packages\pyspark\context.py in _ensure_initialized(cls, instance, gateway, conf)
323 with SparkContext._lock:
324 if not SparkContext._gateway:
--> 325 SparkContext._gateway = gateway or launch_gateway(conf)
326 SparkContext._jvm = SparkContext._gateway.jvm
327
~\Anaconda3\lib\site-packages\pyspark\java_gateway.py in launch_gateway(conf, popen_kwargs)
96 else:
97 # preexec_fn not supported on Windows
---> 98 proc = Popen(command, **popen_kwargs)
99
100 # Wait for the file to appear, or for the process to exit, whichever happens first.
~\Anaconda3\lib\subprocess.py in __init__(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags, restore_signals, start_new_session, pass_fds, encoding, errors, text)
852 encoding=encoding, errors=errors)
853
--> 854 self._execute_child(args, executable, preexec_fn, close_fds,
855 pass_fds, cwd, env,
856 startupinfo, creationflags, shell,
~\Anaconda3\lib\subprocess.py in _execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, unused_restore_signals, unused_start_new_session)
1305 # Start the process
1306 try:
-> 1307 hp, ht, pid, tid = _winapi.CreateProcess(executable, args,
1308 # no special security
1309 None, None,
FileNotFoundError: [WinError 2] The system cannot find the file specified
Sources
This article follows the attribution requirements of Stack Overflow and is licensed under CC BY-SA 3.0.
Source: Stack Overflow
| Solution | Source |
|---|
